From 7887306998198ca7471953a590ae5d127792436b Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:29:20 +0200 Subject: [PATCH 1/4] remove unused line that's now accomplished in subroutine --- pypiper/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypiper/manager.py b/pypiper/manager.py index 965945b..da12953 100644 --- a/pypiper/manager.py +++ b/pypiper/manager.py @@ -1290,7 +1290,6 @@ def proc_wrapup(i): sleeptime = min((sleeptime + 0.25) * 3, 60 / len(processes)) # All jobs are done, print a final closing and job info - stop_time = time.time() proc_message = "Command completed. {info}" info = ( "Elapsed time: " From 07fb60f474c4e3e330172ad53bfc6e21e1346bb6 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:30:01 +0200 Subject: [PATCH 2/4] remove only-once-used variable binding --- pypiper/manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pypiper/manager.py b/pypiper/manager.py index da12953..657dd39 100644 --- a/pypiper/manager.py +++ b/pypiper/manager.py @@ -1290,7 +1290,6 @@ def proc_wrapup(i): sleeptime = min((sleeptime + 0.25) * 3, 60 / len(processes)) # All jobs are done, print a final closing and job info - proc_message = "Command completed. {info}" info = ( "Elapsed time: " + str(datetime.timedelta(seconds=self.time_elapsed(start_time))) @@ -1307,7 +1306,7 @@ def proc_wrapup(i): info += "\n" # finish out the self.info("") - self.info(proc_message.format(info=info)) + self.info("Command completed. {info}".format(info=info)) for rc in returncodes: if rc != 0: From 90fb7ad532b3f954fc649fd99ad33dc318a01f56 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:32:46 +0200 Subject: [PATCH 3/4] allow Stage to be nofail, close #218 --- pypiper/pipeline.py | 5 ++++- pypiper/stage.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pypiper/pipeline.py b/pypiper/pipeline.py index 95d31f2..ab3f8d9 100644 --- a/pypiper/pipeline.py +++ b/pypiper/pipeline.py @@ -332,7 +332,10 @@ def run(self, start_point=None, stop_before=None, stop_after=None): print(f"Running stage: {getattr(stage, 'name', str(stage))}") - stage.run() + try: + stage.run() + except Exception as e: + self.manager._triage_error(e, nofail=stage.nofail) self.executed.append(stage) self.checkpoint(stage) diff --git a/pypiper/stage.py b/pypiper/stage.py index 6f1d551..929becf 100644 --- a/pypiper/stage.py +++ b/pypiper/stage.py @@ -17,7 +17,16 @@ class Stage(object): collection of commands that is checkpointed. """ - def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True): + def __init__( + self, + func, + f_args=None, + f_kwargs=None, + name=None, + checkpoint=True, + *, + nofail=False + ): """ A function, perhaps with arguments, defines the stage. @@ -26,6 +35,8 @@ def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True) :param dict f_kwargs: Keyword arguments for func :param str name: name for the phase/stage :param callable func: Object that defines how the stage will execute. + :param bool nofail: Allow a failure of this stage to not fail the pipeline + in which it's running """ if isinstance(func, Stage): raise TypeError("Cannot create Stage from Stage") @@ -35,6 +46,7 @@ def __init__(self, func, f_args=None, f_kwargs=None, name=None, checkpoint=True) self.f_kwargs = f_kwargs or dict() self.name = name or func.__name__ self.checkpoint = checkpoint + self.nofail = nofail @property def checkpoint_name(self): From 7fa5a67181b589f0e67c68090dc15b72ae7d39c2 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Sun, 5 May 2024 02:39:48 +0200 Subject: [PATCH 4/4] add a requirements file tracking extra dev dependencies --- requirements/requirements-dev-extra.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements/requirements-dev-extra.txt diff --git a/requirements/requirements-dev-extra.txt b/requirements/requirements-dev-extra.txt new file mode 100644 index 0000000..7e66a17 --- /dev/null +++ b/requirements/requirements-dev-extra.txt @@ -0,0 +1 @@ +black