From 2ab4936fb349c021401051f45b5e927bbd375282 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Thu, 28 Nov 2024 04:03:38 +0000
Subject: [PATCH 1/9] [Collector] Fix skipping testcase download when it's not
 found

---
 Collector/Collector.py            | 14 +++++++++++---
 Collector/tests/test_Collector.py |  2 +-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index 2ccf225ce..d240241e3 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -280,8 +280,12 @@ def download(self, crashId):
         if not isinstance(resp_json, dict):
             raise RuntimeError(f"Server sent malformed JSON response: {resp_json!r}")
 
-        if not resp_json["testcase"]:
-            return None
+        if "testcase" not in resp_json or resp_json["testcase"] == "":
+            print(
+                f"Testcase not found for crash {resp_json.get('id', '[no ID]')}",
+                file=sys.stderr,
+            )
+            return (None, resp_json)
 
         response = self.get(dlurl)
 
@@ -324,7 +328,11 @@ def download_all(self, bucketId):
             params = None
 
             for crash in resp_json["results"]:
-                if not crash["testcase"]:
+                if "testcase" not in crash:
+                    print(
+                        f"Testcase not found for crash {crash.get('id', '[no ID]')}",
+                        file=sys.stderr,
+                    )
                     continue
 
                 url = "%s://%s:%d/crashmanager/rest/crashes/%s/download/" % (
diff --git a/Collector/tests/test_Collector.py b/Collector/tests/test_Collector.py
index 0ea43cc97..d572549e5 100644
--- a/Collector/tests/test_Collector.py
+++ b/Collector/tests/test_Collector.py
@@ -390,7 +390,7 @@ def json(self):
 
     collector._session.get = myget1
     result = collector.download(123)
-    assert result is None
+    assert result == (None, {"testcase": ""})
 
     # invalid REST response
     class response1_t:  # noqa

From f52bcd3b97d5265bf845d9731bb288bda0e90b72 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Fri, 29 Nov 2024 17:43:41 +0000
Subject: [PATCH 2/9] [Collector] Add .bin to local_filename when downloading
 crashes without extension

---
 Collector/Collector.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index d240241e3..fa5812f71 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -293,6 +293,12 @@ def download(self, crashId):
             raise RuntimeError(f"Server sent malformed response: {response!r}")
 
         local_filename = f"{crashId}{os.path.splitext(resp_json['testcase'])[1]}"
+        local_filename += (
+            "bin"
+            if resp_json.get("testcase_isbinary") and local_filename.endswith(".")
+            else ""
+        )
+
         with open(local_filename, "wb") as output:
             output.write(response.content)
 

From 8236d25690f84e486fbddf5455482d9f94155f92 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Fri, 29 Nov 2024 17:58:36 +0000
Subject: [PATCH 3/9] [Collector] Refactor download_all to call download

---
 Collector/Collector.py | 49 ++++++++++++++++--------------------------
 1 file changed, 19 insertions(+), 30 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index fa5812f71..a618e89eb 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -250,12 +250,14 @@ def generate(
         return self.__store_signature_hashed(sig)
 
     @remote_checks
-    def download(self, crashId):
+    def download(self, crashId, crashJson=None):
         """
         Download the testcase for the specified crashId.
 
         @type crashId: int
         @param crashId: ID of the requested crash entry on the server side
+        @type crashJson: dict
+        @param crashJson: (optional) FM crash data to skip requesting it here
 
         @rtype: tuple
         @return: Tuple containing name of the file where the test was stored and the raw
@@ -274,11 +276,14 @@ def download(self, crashId):
             self.serverPort,
             crashId,
         )
-
-        resp_json = self.get(url).json()
-
-        if not isinstance(resp_json, dict):
-            raise RuntimeError(f"Server sent malformed JSON response: {resp_json!r}")
+        if not crashJson:
+            resp_json = self.get(url).json()
+            if not isinstance(resp_json, dict):
+                raise RuntimeError(
+                    f"Server sent malformed JSON response: {resp_json!r}"
+                )
+        else:
+            resp_json = crashJson
 
         if "testcase" not in resp_json or resp_json["testcase"] == "":
             print(
@@ -329,35 +334,19 @@ def download_all(self, bucketId):
                 raise RuntimeError(
                     f"Server sent malformed JSON response: {resp_json!r}"
                 )
+            if resp_json["count"] == 0:
+                print("Crashes not found for the specified params.", file=sys.stderr)
+                continue
 
             next_url = resp_json["next"]
             params = None
 
             for crash in resp_json["results"]:
-                if "testcase" not in crash:
-                    print(
-                        f"Testcase not found for crash {crash.get('id', '[no ID]')}",
-                        file=sys.stderr,
-                    )
-                    continue
-
-                url = "%s://%s:%d/crashmanager/rest/crashes/%s/download/" % (
-                    self.serverProtocol,
-                    self.serverHost,
-                    self.serverPort,
-                    crash["id"],
-                )
-                response = self.get(url)
-
-                if "content-disposition" not in response.headers:
-                    raise RuntimeError(f"Server sent malformed response: {response!r}")
+                # crash here must already have same data as individual resp by crash ID
+                (local_filename, resp_dl) = self.download(crash["id"], crash)
 
-                local_filename = "%d%s" % (
-                    crash["id"],
-                    os.path.splitext(crash["testcase"])[1],
-                )
-                with open(local_filename, "wb") as output:
-                    output.write(response.content)
+                if not local_filename:
+                    continue
 
                 yield local_filename
 
@@ -798,7 +787,7 @@ def main(args=None):
 
         for result in collector.download_all(opts.download_all):
             downloaded = True
-            print(result)
+            print("Downloaded: ", result)
 
         if not downloaded:
             print("Specified signature does not have any testcases", file=sys.stderr)

From 12bcf6165aa7aa1669ed81d9d01aa036ea949448 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Fri, 29 Nov 2024 22:40:36 +0000
Subject: [PATCH 4/9] [Collector] Download testcases by any query_params

---
 Collector/Collector.py | 112 +++++++++++++++++++++++++++++++++--------
 1 file changed, 91 insertions(+), 21 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index a618e89eb..f10d4fae9 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -37,6 +37,20 @@
 __updated__ = "2014-10-01"
 
 
+class KeyValueAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        result = {}
+        for item in values:
+            try:
+                key, value = item.split("=")
+                result[key] = value
+            except ValueError:
+                raise argparse.ArgumentTypeError(
+                    f"Invalid format: '{item}', expected key=value."
+                )
+        setattr(namespace, self.dest, result)
+
+
 class Collector(Reporter):
     @remote_checks
     @signature_checks
@@ -310,45 +324,71 @@ def download(self, crashId, crashJson=None):
         return (local_filename, resp_json)
 
     @remote_checks
-    def download_all(self, bucketId):
+    def download_all(self, query_params):
         """
-        Download all testcases for the specified bucketId.
+        Download all testcases for the specified params.
 
-        @type bucketId: int
-        @param bucketId: ID of the requested bucket on the server side
+        @type params: dict
+        @param params: dictionary of params to download testcases for
 
         @rtype: generator
         @return: generator of filenames where tests were stored.
         """
-        params = {"query": json.dumps({"op": "OR", "bucket": bucketId})}
-        next_url = "%s://%s:%d/crashmanager/rest/crashes/" % (
+        # iterate over each page
+        for response in self.get_by_query("crashes/", query_params):
+            resp_json = response
+
+            for crash in resp_json["results"]:
+                # crash here must already have same data as individual resp by crash ID
+                (local_filename, resp_dl) = self.download(crash["id"], crash)
+                if not local_filename:
+                    continue
+
+                yield local_filename
+
+    @remote_checks
+    def get_by_query(self, rest_endpoint, query_params={}):
+        """
+        Get request for the specified REST endpoint and query params.
+
+        @type rest_endpoint: str
+        @param rest_endpoint: for crashmanager/rest/{rest_endpoint}.
+
+        @type params: dict
+        @param params: dictionary of params to query with; empty default.
+
+        @rtype: generator
+        @return: generator of JSON responses for the specified query.
+        """
+        url_rest = "%s://%s:%d/crashmanager/rest/" % (
             self.serverProtocol,
             self.serverHost,
             self.serverPort,
         )
+        next_url = url_rest + rest_endpoint
+        params = {"query": json.dumps({"op": "AND", **query_params})}
 
         while next_url:
             resp_json = self.get(next_url, params=params).json()
-
-            if not isinstance(resp_json, dict):
+            if not (isinstance(resp_json, dict) or isinstance(resp_json[0], dict)):
                 raise RuntimeError(
                     f"Server sent malformed JSON response: {resp_json!r}"
                 )
-            if resp_json["count"] == 0:
-                print("Crashes not found for the specified params.", file=sys.stderr)
+            if len(resp_json) == 0 or (
+                isinstance(resp_json, dict) and resp_json.get("count") == 0
+            ):
+                print(
+                    f"Results not found for {query_params} at {next_url}.",
+                    file=sys.stderr,
+                )
                 continue
 
-            next_url = resp_json["next"]
+            if isinstance(resp_json, list):
+                next_url = resp_json[0].get("next")
+            else:
+                next_url = resp_json.get("next")
             params = None
-
-            for crash in resp_json["results"]:
-                # crash here must already have same data as individual resp by crash ID
-                (local_filename, resp_dl) = self.download(crash["id"], crash)
-
-                if not local_filename:
-                    continue
-
-                yield local_filename
+            yield resp_json
 
     def __store_signature_hashed(self, signature):
         """
@@ -455,6 +495,11 @@ def main(args=None):
         help="Download all testcases for the specified signature entry",
         metavar="ID",
     )
+    actions.add_argument(
+        "--download-by-params",
+        action="store_true",
+        help="Download all testcases for the crashes specified by --query-params",
+    )
     actions.add_argument(
         "--get-clientid",
         action="store_true",
@@ -516,9 +561,20 @@ def main(args=None):
     parser.add_argument(
         "--env",
         nargs="+",
+        # action=KeyValueAction, #TODO: my action instead of manual key=value splitting?
         type=str,
         help="List of environment variables in the form 'KEY=VALUE'",
     )
+    parser.add_argument(
+        "--query-params",
+        nargs="+",
+        action=KeyValueAction,
+        help="""Specify query params (key=value) to download/resubmit crashes:
+        args, bucket, bucket_id, cachedCrashInfo, client, client_id, crashAddress,
+        crashAddressNumeric, created, env, id, metadata, os, os_id, platform,
+        platform_id, product, product_id, rawCrashData, rawStderr, rawStdout,
+        shortSignature, testcase, testcase_id, tool, tool_id, triagedOnce""",
+    )
     parser.add_argument(
         "--metadata",
         nargs="+",
@@ -785,7 +841,7 @@ def main(args=None):
     if opts.download_all:
         downloaded = False
 
-        for result in collector.download_all(opts.download_all):
+        for result in collector.download_all({"bucket": opts.download_all}):
             downloaded = True
             print("Downloaded: ", result)
 
@@ -795,6 +851,20 @@ def main(args=None):
 
         return 0
 
+    if opts.download_by_params:
+        downloaded = False
+
+        if not opts.query_params:
+            print("Specify query params to download testcases", file=sys.stderr)
+            return 1
+
+        for result in collector.download_all(opts.query_params):
+            downloaded = True
+            print("Downloaded: ", result)
+
+        if not downloaded:
+            print("Failed to download testcases for the specified params")
+
     if opts.get_clientid:
         print(collector.clientId)
         return 0

From 5539861d1091ff0d411608b5aa199d1a67662511 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Thu, 5 Dec 2024 15:03:52 +0000
Subject: [PATCH 5/9] [Collector] Add refresh-crashes option

---
 Collector/Collector.py | 55 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 6 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index f10d4fae9..a74d9770e 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -500,6 +500,11 @@ def main(args=None):
         action="store_true",
         help="Download all testcases for the crashes specified by --query-params",
     )
+    actions.add_argument(
+        "--refresh-crashes",
+        action="store_true",
+        help="Refresh (download, resubmit) all testcases specified by --query-params",
+    )
     actions.add_argument(
         "--get-clientid",
         action="store_true",
@@ -628,13 +633,13 @@ def main(args=None):
     # In autosubmit mode, we try to open a configuration file for the binary specified
     # on the command line. It should contain the binary-specific settings for
     # submitting.
-    if opts.autosubmit:
-        if not opts.rargs:
-            parser.error("Action --autosubmit requires test arguments to be specified")
-
+    if opts.autosubmit or opts.refresh_crashes:
         # Store the binary candidate only if --binary wasn't also specified
         if not opts.binary:
             opts.binary = opts.rargs[0]
+    if opts.autosubmit:
+        if not opts.rargs:
+            parser.error("Action --autosubmit requires test arguments to be specified")
 
         # We also need to check that (apart from the binary), there is only one file on
         # the command line (the testcase), if it hasn't been explicitly specified.
@@ -664,7 +669,13 @@ def main(args=None):
     env = None
     metadata = {}
 
-    if opts.search or opts.generate or opts.submit or opts.autosubmit:
+    if (
+        opts.search
+        or opts.generate
+        or opts.submit
+        or opts.autosubmit
+        or opts.refresh_crashes
+    ):
         if opts.metadata:
             metadata.update(dict(kv.split("=", 1) for kv in opts.metadata))
 
@@ -717,7 +728,7 @@ def main(args=None):
                 metadata,
             )
 
-        if not opts.autosubmit:
+        if not opts.autosubmit and not opts.refresh_crashes:
             if opts.stderr is None and opts.crashdata is None:
                 parser.error(
                     "Must specify at least either --stderr or --crashdata file"
@@ -865,6 +876,38 @@ def main(args=None):
         if not downloaded:
             print("Failed to download testcases for the specified params")
 
+    if opts.refresh_crashes:
+        if not opts.query_params:
+            print("Specify query params to refresh crashes.", file=sys.stderr)
+            return 1
+        for testcase in collector.download_all(opts.query_params):
+            # TODO: support positional testcases (e.g. for libfuzzer)
+            os.environ["MOZ_FUZZ_TESTFILE"] = testcase  # needed by AFL++
+            runner = AutoRunner.fromBinaryArgs(opts.binary, opts.rargs[1:])
+            # TODO: diff old vs new crash?
+            ran = runner.run()
+            if ran:
+                crashInfo = runner.getCrashInfo(configuration)
+                submission = collector.submit(
+                    crashInfo,
+                    testcase,
+                    opts.testcasequality,
+                    opts.testcasesize,
+                    metadata,
+                )
+                print(
+                    "Resubmitted old crash {} as {}.".format(
+                        os.path.splitext(testcase)[0], submission["id"]
+                    )
+                )
+            else:
+                print(
+                    "Error: Failed to reproduce crash %s, can't submit."
+                    % os.path.splitext(testcase)[0],
+                    file=sys.stderr,
+                )
+        return 0
+
     if opts.get_clientid:
         print(collector.clientId)
         return 0

From f65584128d863cc0e48cbfa42a9461511315fd22 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Thu, 5 Dec 2024 23:41:43 +0000
Subject: [PATCH 6/9] [Collector] Refactor key=value parsing for env, metadata

---
 Collector/Collector.py | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index a74d9770e..054ebacb1 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -42,7 +42,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         result = {}
         for item in values:
             try:
-                key, value = item.split("=")
+                key, value = item.split("=", 1)
                 result[key] = value
             except ValueError:
                 raise argparse.ArgumentTypeError(
@@ -566,8 +566,7 @@ def main(args=None):
     parser.add_argument(
         "--env",
         nargs="+",
-        # action=KeyValueAction, #TODO: my action instead of manual key=value splitting?
-        type=str,
+        action=KeyValueAction,
         help="List of environment variables in the form 'KEY=VALUE'",
     )
     parser.add_argument(
@@ -583,7 +582,7 @@ def main(args=None):
     parser.add_argument(
         "--metadata",
         nargs="+",
-        type=str,
+        action=KeyValueAction,
         help="List of metadata variables in the form 'KEY=VALUE'",
     )
     parser.add_argument(
@@ -666,8 +665,7 @@ def main(args=None):
     crashdata = None
     crashInfo = None
     args = None
-    env = None
-    metadata = {}
+    metadata = opts.metadata if opts.metadata else {}
 
     if (
         opts.search
@@ -676,9 +674,6 @@ def main(args=None):
         or opts.autosubmit
         or opts.refresh_crashes
     ):
-        if opts.metadata:
-            metadata.update(dict(kv.split("=", 1) for kv in opts.metadata))
-
         if opts.autosubmit:
             # Try to automatically get arguments from the command line
             # If the testcase is not the last argument, leave it in the
@@ -693,9 +688,6 @@ def main(args=None):
             if opts.args:
                 args = [arg.replace("\\", "") for arg in opts.args]
 
-        if opts.env:
-            env = dict(kv.split("=", 1) for kv in opts.env)
-
         # Start without any ProgramConfiguration
         configuration = None
 
@@ -703,8 +695,8 @@ def main(args=None):
         if opts.binary:
             configuration = ProgramConfiguration.fromBinary(opts.binary)
             if configuration:
-                if env:
-                    configuration.addEnvironmentVariables(env)
+                if opts.env:
+                    configuration.addEnvironmentVariables(opts.env)
                 if args:
                     configuration.addProgramArguments(args)
                 if metadata:
@@ -723,7 +715,7 @@ def main(args=None):
                 opts.platform,
                 opts.os,
                 opts.product_version,
-                env,
+                opts.env,
                 args,
                 metadata,
             )
@@ -832,10 +824,10 @@ def main(args=None):
             print("")
 
         if "env" in retJSON and retJSON["env"]:
-            env = json.loads(retJSON["env"])
+            opts.env = json.loads(retJSON["env"])
             print(
                 "Environment variables:",
-                " ".join(f"{k} = {v}" for (k, v) in env.items()),
+                " ".join(f"{k} = {v}" for (k, v) in opts.env.items()),
             )
             print("")
 

From 01fec96630261b2830f4947763c47d7b661cd0bf Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Tue, 10 Dec 2024 22:33:42 +0000
Subject: [PATCH 7/9] [Collector] Specify all tool filter buckets with
 bucket=MYBUCKETS for download/refresh

---
 Collector/Collector.py | 87 ++++++++++++++++++++++++++++--------------
 1 file changed, 58 insertions(+), 29 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index 054ebacb1..66c4a4336 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -577,7 +577,8 @@ def main(args=None):
         args, bucket, bucket_id, cachedCrashInfo, client, client_id, crashAddress,
         crashAddressNumeric, created, env, id, metadata, os, os_id, platform,
         platform_id, product, product_id, rawCrashData, rawStderr, rawStdout,
-        shortSignature, testcase, testcase_id, tool, tool_id, triagedOnce""",
+        shortSignature, testcase, testcase_id, tool, tool_id, triagedOnce. Instead, you
+        can also get all crashes in your buckets (tool filter) with bucket=MYBUCKETS""",
     )
     parser.add_argument(
         "--metadata",
@@ -760,6 +761,11 @@ def main(args=None):
         opts.clientid,
         opts.tool,
     )
+    url_rest = "%s://%s:%d/crashmanager/rest/" % (
+        collector.serverProtocol,
+        collector.serverHost,
+        collector.serverPort,
+    )
 
     if opts.refresh:
         collector.refresh()
@@ -854,6 +860,26 @@ def main(args=None):
 
         return 0
 
+    if opts.query_params.get("bucket") == "MYBUCKETS":
+        if len(opts.query_params) > 1:
+            print("Can't use other query params w/ bucket=MYBUCKETS", file=sys.stderr)
+            return 1
+        # ignore all other query params when mybuckets is used
+        nobuckets = True
+        buckets = set()
+        for response in collector.get_by_query("buckets/"):
+            if len(response) > 0:
+                nobuckets = False
+            for bucket in response:
+                buckets.add(bucket["id"])
+        if nobuckets:
+            print("No buckets found", file=sys.stderr)
+            return 1
+        all_params = [{"bucket": bucket} for bucket in buckets]
+        # TODO: some check if no toolfilter is set?
+    elif opts.query_params:
+        all_params = [opts.query_params]
+
     if opts.download_by_params:
         downloaded = False
 
@@ -861,9 +887,10 @@ def main(args=None):
             print("Specify query params to download testcases", file=sys.stderr)
             return 1
 
-        for result in collector.download_all(opts.query_params):
-            downloaded = True
-            print("Downloaded: ", result)
+        for param in all_params:
+            for result in collector.download_all(param):
+                downloaded = True
+                print("Downloaded: ", result)
 
         if not downloaded:
             print("Failed to download testcases for the specified params")
@@ -872,32 +899,34 @@ def main(args=None):
         if not opts.query_params:
             print("Specify query params to refresh crashes.", file=sys.stderr)
             return 1
-        for testcase in collector.download_all(opts.query_params):
-            # TODO: support positional testcases (e.g. for libfuzzer)
-            os.environ["MOZ_FUZZ_TESTFILE"] = testcase  # needed by AFL++
-            runner = AutoRunner.fromBinaryArgs(opts.binary, opts.rargs[1:])
-            # TODO: diff old vs new crash?
-            ran = runner.run()
-            if ran:
-                crashInfo = runner.getCrashInfo(configuration)
-                submission = collector.submit(
-                    crashInfo,
-                    testcase,
-                    opts.testcasequality,
-                    opts.testcasesize,
-                    metadata,
-                )
-                print(
-                    "Resubmitted old crash {} as {}.".format(
-                        os.path.splitext(testcase)[0], submission["id"]
+
+        for param in all_params:
+            for testcase in collector.download_all(param):
+                # TODO: support positional testcases (e.g. for libfuzzer)
+                os.environ["MOZ_FUZZ_TESTFILE"] = testcase  # needed by AFL++
+                runner = AutoRunner.fromBinaryArgs(opts.binary, opts.rargs[1:])
+                # TODO: diff old vs new crash?
+                ran = runner.run()
+                if ran:
+                    crashInfo = runner.getCrashInfo(configuration)
+                    submission = collector.submit(
+                        crashInfo,
+                        testcase,
+                        opts.testcasequality,
+                        opts.testcasesize,
+                        metadata,
+                    )
+                    print(
+                        "Resubmitted old crash {} as {}.".format(
+                            os.path.splitext(testcase)[0], submission["id"]
+                        )
+                    )
+                else:
+                    print(
+                        "Error: Failed to reproduce crash %s, can't submit."
+                        % os.path.splitext(testcase)[0],
+                        file=sys.stderr,
                     )
-                )
-            else:
-                print(
-                    "Error: Failed to reproduce crash %s, can't submit."
-                    % os.path.splitext(testcase)[0],
-                    file=sys.stderr,
-                )
         return 0
 
     if opts.get_clientid:

From a1c90b64190cf65c3058e9554bab00333b36803f Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Fri, 27 Dec 2024 14:54:07 +0000
Subject: [PATCH 8/9] [Collector] Add --best-entry-only option for
 downloading/refreshing crashes

---
 Collector/Collector.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index 66c4a4336..e7ad5fa9d 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -580,6 +580,11 @@ def main(args=None):
         shortSignature, testcase, testcase_id, tool, tool_id, triagedOnce. Instead, you
         can also get all crashes in your buckets (tool filter) with bucket=MYBUCKETS""",
     )
+    parser.add_argument(
+        "--best-entry-only",
+        action="store_true",
+        help="Refresh only the best entry crashes for buckets found by --query-params",
+    )
     parser.add_argument(
         "--metadata",
         nargs="+",
@@ -878,7 +883,19 @@ def main(args=None):
         all_params = [{"bucket": bucket} for bucket in buckets]
         # TODO: some check if no toolfilter is set?
     elif opts.query_params:
-        all_params = [opts.query_params]
+        if opts.best_entry_only:
+            buckets = set()
+            for response in collector.get_by_query("crashes/", opts.query_params):
+                for crash in response["results"]:
+                    buckets.add(crash["bucket"])
+        else:
+            all_params = [opts.query_params]
+
+    if opts.best_entry_only:
+        all_params = []
+        for bucket in buckets:
+            resp_bucket = collector.get(url_rest + f"buckets/{bucket}").json()
+            all_params.append({"bucket": bucket, "id": resp_bucket["best_entry"]})
 
     if opts.download_by_params:
         downloaded = False

From 3e026eedc0519a7df4b2ec39e97d0db1a1939848 Mon Sep 17 00:00:00 2001
From: asuleimanov <asuleimanov@mozilla.com>
Date: Fri, 27 Dec 2024 19:47:21 +0000
Subject: [PATCH 9/9] [Collector] Add --ignore-toolfilter option for
 downloading/refreshing crashes

---
 Collector/Collector.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/Collector/Collector.py b/Collector/Collector.py
index e7ad5fa9d..58c425477 100755
--- a/Collector/Collector.py
+++ b/Collector/Collector.py
@@ -347,16 +347,18 @@ def download_all(self, query_params):
                 yield local_filename
 
     @remote_checks
-    def get_by_query(self, rest_endpoint, query_params={}):
+    def get_by_query(self, rest_endpoint, query_params={}, _ignore_toolfilter=None):
         """
         Get request for the specified REST endpoint and query params.
 
         @type rest_endpoint: str
         @param rest_endpoint: for crashmanager/rest/{rest_endpoint}.
 
-        @type params: dict
-        @param params: dictionary of params to query with; empty default.
+        @type query_params: dict
+        @param query_params: dictionary of params to query with; empty default.
 
+        @type _ignore_toolfilter: int
+        @param _ignore_toolfilter: integer 0 or 1 to ignore your set toolfilter
         @rtype: generator
         @return: generator of JSON responses for the specified query.
         """
@@ -366,7 +368,14 @@ def get_by_query(self, rest_endpoint, query_params={}):
             self.serverPort,
         )
         next_url = url_rest + rest_endpoint
-        params = {"query": json.dumps({"op": "AND", **query_params})}
+        global ignore_toolfilter
+        ignore = (
+            _ignore_toolfilter if _ignore_toolfilter is not None else ignore_toolfilter
+        )
+        params = {
+            "query": json.dumps({"op": "AND", **query_params}),
+            "ignore_toolfilter": ignore,
+        }
 
         while next_url:
             resp_json = self.get(next_url, params=params).json()
@@ -585,6 +594,11 @@ def main(args=None):
         action="store_true",
         help="Refresh only the best entry crashes for buckets found by --query-params",
     )
+    parser.add_argument(
+        "--ignore-toolfilter",
+        action="store_true",
+        help="Ignore your (supposedly) set toolfilter for queries.",
+    )
     parser.add_argument(
         "--metadata",
         nargs="+",
@@ -771,6 +785,8 @@ def main(args=None):
         collector.serverHost,
         collector.serverPort,
     )
+    global ignore_toolfilter
+    ignore_toolfilter = 1 if opts.ignore_toolfilter else 0
 
     if opts.refresh:
         collector.refresh()
@@ -881,7 +897,6 @@ def main(args=None):
             print("No buckets found", file=sys.stderr)
             return 1
         all_params = [{"bucket": bucket} for bucket in buckets]
-        # TODO: some check if no toolfilter is set?
     elif opts.query_params:
         if opts.best_entry_only:
             buckets = set()
@@ -894,7 +909,10 @@ def main(args=None):
     if opts.best_entry_only:
         all_params = []
         for bucket in buckets:
-            resp_bucket = collector.get(url_rest + f"buckets/{bucket}").json()
+            resp_bucket = collector.get(
+                url_rest + f"buckets/{bucket}",
+                params={"ignore_toolfilter": ignore_toolfilter},
+            ).json()
             all_params.append({"bucket": bucket, "id": resp_bucket["best_entry"]})
 
     if opts.download_by_params: