feat(quarterstack): improve flexibility of quarterstack revision crea…

…tion

ch_pipeline/processing/quarterstack.py

@@ -8,6 +8,7 @@
 """
 
 import re
+import warnings
 from typing import ClassVar
 
 import numpy as np
@@ -353,23 +354,23 @@ class QuarterStackProcessing(base.ProcessingType):
     default_params: ClassVar = {
         # Daily processing revisions to use (later entries in this list take precedence
         # over earlier ones)
-        "daily_revisions": ["rev_07"],
+        "daily_revisions": ["rev_08"],
         # Usually the opinions are queried for each revision, this dictionary allows
         # that to be overridden. Each `data_rev: opinion_rev` pair means that the
         # opinions used to select days for `data_rev` will instead be taken from
         # `opinion_rev`.
         "opinion_overrides": {
             "rev_03": "rev_02",
         },
-        "daily_root": "/project/rpp-chime/chime/chime_processed/",
+        "daily_root": None,
         # Frequencies to process
         "freq": [0, 1024],
         "nfreq_delay": 1025,
         # The beam transfers to use (need to have the same freq range as above)
         "product_path": "/project/rpp-chime/chime/bt_empty/chime_4cyl_allfreq/",
         # System modules to use/load
         "modpath": "/project/rpp-chime/chime/chime_env/modules/modulefiles",
-        "modlist": "chime/python/2022.06",
+        "modlist": "chime/python/2024.04",
         "partitions": 2,
         # Don't generate quarter stacks with less days than this
         "min_days": 5,
@@ -418,50 +419,64 @@ def _create_hook(self):
         This tries to determine which days are good and bad, and partitions the
         available good days into the individual stacks.
         """
+        # Request additional information from the user
+        daily_revs = input(
+            "Enter the daily revisions to include (<rev_ij>,<rev_ik>,...): "
+        )
+        if daily_revs:
+            daily_revs = re.compile(r"rev_[0-9]{2}").findall(daily_revs)
+            for d in daily_revs:
+                if d not in self.default_params["daily_revisions"]:
+                    self.default_params["daily_revisions"].append(d)
+
         days = {}
 
         core.connect()
 
         opinion_overrides = self.default_params.get("opinion_overrides", {})
 
         # Go over each revision and construct the set of LSDs we should stack, and save
-        # the path to each.
-        # NOTE: later entries in `daily_revisions` will override LSDs found in earlier
-        # revisions.
+        # the path to each. Later entries in `daily_revisions` will override LSDs found
+        # in earlier revisions.
         for rev in self.default_params["daily_revisions"]:
             daily_path = (
                 self.root_path
                 if self.default_params["daily_root"] is None
                 else self.default_params["daily_root"]
             )
-            daily_rev = daily.DailyProcessing(rev, root_path=daily_path)
+            try:
+                daily_rev = daily.DailyProcessing(rev, root_path=daily_path)
+            except Exception:  # noqa: BLE001
+                warnings.warn(f"Could not load revision {rev} at '{daily_path}'")
+                continue
 
             # Get the revision used to determine the opinions, by default this is the
             # revision, but it can be overriden
             opinion_rev = opinion_overrides.get(rev, rev)
 
-            # Get all the bad days in this revision
-            revision = df.DataRevision.get(name=opinion_rev)
-            query = (
-                df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
-                .distinct()
-                .where(
-                    df.DataFlagOpinion.revision == revision,
-                    df.DataFlagOpinion.decision == "bad",
+            if opinion_rev is not None:
+                # Get all the bad days in this revision
+                revision = df.DataRevision.get(name=opinion_rev)
+                query = (
+                    df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
+                    .distinct()
+                    .where(
+                        df.DataFlagOpinion.revision == revision,
+                        df.DataFlagOpinion.decision == "bad",
+                    )
                 )
-            )
-            bad_days = [x[0] for x in query.tuples()]
-
-            # Get all the good days
-            query = (
-                df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
-                .distinct()
-                .where(
-                    df.DataFlagOpinion.revision == revision,
-                    df.DataFlagOpinion.decision == "good",
+                bad_days = [x[0] for x in query.tuples()]
+
+                # Get all the good days
+                query = (
+                    df.DataFlagOpinion.select(df.DataFlagOpinion.lsd)
+                    .distinct()
+                    .where(
+                        df.DataFlagOpinion.revision == revision,
+                        df.DataFlagOpinion.decision == "good",
+                    )
                 )
-            )
-            good_days = [x[0] for x in query.tuples()]
+                good_days = [x[0] for x in query.tuples()]
 
             for d in daily_rev.ls():
                 try:
@@ -471,9 +486,12 @@ def _create_hook(self):
                         f'Could not parse string tag "{d}" into a valid LSD'
                     ) from e
 
-                # Filter out known bad days here
-                if (lsd in bad_days) or (lsd not in good_days):
-                    continue
+                # Filter out known bad days here. If `opinion_rev` is None,
+                # ignore opinions and automatically include all available days.
+                # This is only true if the opinion override is explicitly set
+                if opinion_rev is not None:
+                    if (lsd in bad_days) or (lsd not in good_days):
+                        continue
 
                 # Insert the day and path into the dict, this will replace the entries
                 # from prior revisions