diff --git a/ch_pipeline/processing/quarterstack.py b/ch_pipeline/processing/quarterstack.py index 161f5625..862e3236 100644 --- a/ch_pipeline/processing/quarterstack.py +++ b/ch_pipeline/processing/quarterstack.py @@ -8,6 +8,7 @@ """ import re +import warnings from typing import ClassVar import numpy as np @@ -353,7 +354,7 @@ class QuarterStackProcessing(base.ProcessingType): default_params: ClassVar = { # Daily processing revisions to use (later entries in this list take precedence # over earlier ones) - "daily_revisions": ["rev_07"], + "daily_revisions": ["rev_08"], # Usually the opinions are queried for each revision, this dictionary allows # that to be overridden. Each `data_rev: opinion_rev` pair means that the # opinions used to select days for `data_rev` will instead be taken from @@ -361,7 +362,7 @@ class QuarterStackProcessing(base.ProcessingType): "opinion_overrides": { "rev_03": "rev_02", }, - "daily_root": "/project/rpp-chime/chime/chime_processed/", + "daily_root": None, # Frequencies to process "freq": [0, 1024], "nfreq_delay": 1025, @@ -369,7 +370,7 @@ class QuarterStackProcessing(base.ProcessingType): "product_path": "/project/rpp-chime/chime/bt_empty/chime_4cyl_allfreq/", # System modules to use/load "modpath": "/project/rpp-chime/chime/chime_env/modules/modulefiles", - "modlist": "chime/python/2022.06", + "modlist": "chime/python/2024.04", "partitions": 2, # Don't generate quarter stacks with less days than this "min_days": 5, @@ -418,6 +419,16 @@ def _create_hook(self): This tries to determine which days are good and bad, and partitions the available good days into the individual stacks. """ + # Request additional information from the user + daily_revs = input( + "Enter the daily revisions to include (,,...): " + ) + if daily_revs: + daily_revs = re.compile(r"rev_[0-9]{2}").findall(daily_revs) + for d in daily_revs: + if d not in self.default_params["daily_revisions"]: + self.default_params["daily_revisions"].append(d) + days = {} core.connect() @@ -425,43 +436,47 @@ def _create_hook(self): opinion_overrides = self.default_params.get("opinion_overrides", {}) # Go over each revision and construct the set of LSDs we should stack, and save - # the path to each. - # NOTE: later entries in `daily_revisions` will override LSDs found in earlier - # revisions. + # the path to each. Later entries in `daily_revisions` will override LSDs found + # in earlier revisions. for rev in self.default_params["daily_revisions"]: daily_path = ( self.root_path if self.default_params["daily_root"] is None else self.default_params["daily_root"] ) - daily_rev = daily.DailyProcessing(rev, root_path=daily_path) + try: + daily_rev = daily.DailyProcessing(rev, root_path=daily_path) + except Exception: # noqa: BLE001 + warnings.warn(f"Could not load revision {rev} at '{daily_path}'") + continue # Get the revision used to determine the opinions, by default this is the # revision, but it can be overriden opinion_rev = opinion_overrides.get(rev, rev) - # Get all the bad days in this revision - revision = df.DataRevision.get(name=opinion_rev) - query = ( - df.DataFlagOpinion.select(df.DataFlagOpinion.lsd) - .distinct() - .where( - df.DataFlagOpinion.revision == revision, - df.DataFlagOpinion.decision == "bad", + if opinion_rev is not None: + # Get all the bad days in this revision + revision = df.DataRevision.get(name=opinion_rev) + query = ( + df.DataFlagOpinion.select(df.DataFlagOpinion.lsd) + .distinct() + .where( + df.DataFlagOpinion.revision == revision, + df.DataFlagOpinion.decision == "bad", + ) ) - ) - bad_days = [x[0] for x in query.tuples()] - - # Get all the good days - query = ( - df.DataFlagOpinion.select(df.DataFlagOpinion.lsd) - .distinct() - .where( - df.DataFlagOpinion.revision == revision, - df.DataFlagOpinion.decision == "good", + bad_days = [x[0] for x in query.tuples()] + + # Get all the good days + query = ( + df.DataFlagOpinion.select(df.DataFlagOpinion.lsd) + .distinct() + .where( + df.DataFlagOpinion.revision == revision, + df.DataFlagOpinion.decision == "good", + ) ) - ) - good_days = [x[0] for x in query.tuples()] + good_days = [x[0] for x in query.tuples()] for d in daily_rev.ls(): try: @@ -471,9 +486,12 @@ def _create_hook(self): f'Could not parse string tag "{d}" into a valid LSD' ) from e - # Filter out known bad days here - if (lsd in bad_days) or (lsd not in good_days): - continue + # Filter out known bad days here. If `opinion_rev` is None, + # ignore opinions and automatically include all available days. + # This is only true if the opinion override is explicitly set + if opinion_rev is not None: + if (lsd in bad_days) or (lsd not in good_days): + continue # Insert the day and path into the dict, this will replace the entries # from prior revisions