From db82bbb8a184ebeff9ff982a7b8d3b24b7e75621 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 20 May 2022 12:11:21 +0200 Subject: [PATCH] fix empty scatter bug --- cwltool/workflow_job.py | 16 +- tests/test_conditionals.py | 155 ++++++++++++++++++ .../scatter_before_when-flat_crossproduct.cwl | 38 +++++ tests/wf/scatter_before_when-inp.yaml | 6 + tests/wf/scatter_before_when-inp_empty.yaml | 2 + ...catter_before_when-nested_crossproduct.cwl | 38 +++++ tests/wf/scatter_before_when.cwl | 28 ++++ .../scatter_before_when_dotproduct-inp.yaml | 12 ++ tests/wf/scatter_before_when_dotproduct.cwl | 38 +++++ 9 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 tests/wf/scatter_before_when-flat_crossproduct.cwl create mode 100644 tests/wf/scatter_before_when-inp.yaml create mode 100644 tests/wf/scatter_before_when-inp_empty.yaml create mode 100644 tests/wf/scatter_before_when-nested_crossproduct.cwl create mode 100644 tests/wf/scatter_before_when.cwl create mode 100644 tests/wf/scatter_before_when_dotproduct-inp.yaml create mode 100644 tests/wf/scatter_before_when_dotproduct.cwl diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index f28dd08c0..3565e08d7 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -172,7 +172,7 @@ def nested_crossproduct_scatter( runtimeContext: RuntimeContext, ) -> JobsGeneratorType: scatter_key = scatter_keys[0] - jobl = len(cast(Sized, joborder[scatter_key])) + jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0 output = {} # type: ScatterDestinationsType for i in process.tool["outputs"]: output[i["id"]] = [None] * jobl @@ -219,7 +219,9 @@ def crossproduct_size( ssum = len(cast(Sized, joborder[scatter_key])) else: ssum = 0 - for _ in range(0, len(cast(Sized, joborder[scatter_key]))): + for _ in range( + 0, len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0 + ): ssum += crossproduct_size(joborder, scatter_keys[1:]) return ssum @@ -252,7 +254,7 @@ def _flat_crossproduct_scatter( ) -> Tuple[List[Optional[JobsGeneratorType]], int,]: """Inner loop.""" scatter_key = scatter_keys[0] - jobl = len(cast(Sized, joborder[scatter_key])) + jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0 steps = [] # type: List[Optional[JobsGeneratorType]] put = startindex for index in range(0, jobl): @@ -292,8 +294,8 @@ def dotproduct_scatter( jobl = None # type: Optional[int] for key in scatter_keys: if jobl is None: - jobl = len(cast(Sized, joborder[key])) - elif jobl != len(cast(Sized, joborder[key])): + jobl = len(cast(Sized, joborder[key])) if joborder[key] else 0 + elif jobl != len(cast(Sized, joborder[key])) if joborder[key] else 0: raise WorkflowException( "Length of input arrays must be equal when performing " "dotproduct scatter." @@ -729,7 +731,9 @@ def valueFromFunc( runtimeContext.postScatterEval = postScatterEval emptyscatter = [ - shortname(s) for s in scatter if len(cast(Sized, inputobj[s])) == 0 + shortname(s) + for s in scatter + if not inputobj[s] or len(cast(Sized, inputobj[s])) == 0 ] if emptyscatter: _logger.warning( diff --git a/tests/test_conditionals.py b/tests/test_conditionals.py index 8665f1e3a..3baef7b12 100644 --- a/tests/test_conditionals.py +++ b/tests/test_conditionals.py @@ -17,3 +17,158 @@ def test_conditional_step_no_inputs() -> None: stderr = re.sub(r"\s\s+", " ", stderr) assert err_code == 0, stderr assert result is None + + +def test_conditional_scatter_missing_input() -> None: + """Test that scattering a missing array skips execution.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when.cwl"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [] + + +def test_scatter_empty_array() -> None: + """Test that scattering an empty array skips execution.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when.cwl"), + get_data("tests/wf/scatter_before_when-inp_empty.yaml"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [] + + +def test_scatter_and_conditional() -> None: + """Test scattering a partially empty array with a conditional.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when.cwl"), + get_data("tests/wf/scatter_before_when-inp.yaml"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == ["We\n", "come\n", "in\n", "peace\n"] + + +def test_scatter_dotproduct_empty_arrays() -> None: + """Test that dotproduct scattering empty arrays skips execution.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when_dotproduct.cwl"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [] + + +def test_scatter_dotproduct_and_conditional() -> None: + """Test dotproduct scattering with partially empty arrays.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when_dotproduct.cwl"), + get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [ + "We Never\n", + "Come Out\n", + "In Anything But\n", + "Peace -- The Aliens\n", + ] + + +def test_scatter_nested_crossproduct_empty_arrays() -> None: + """Test that nested_dotproduct scattering empty arrays skips execution.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [] + + +def test_scatter_nested_crossproduct_and_conditional() -> None: + """Test nested_crossproduct scattering with partially empty arrays.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"), + get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [ + ["We Never\n", "We Out\n", "We Anything But\n", None, "We -- The Aliens\n"], + [ + "Come Never\n", + "Come Out\n", + "Come Anything But\n", + None, + "Come -- The Aliens\n", + ], + ["In Never\n", "In Out\n", "In Anything But\n", None, "In -- The Aliens\n"], + [None, None, None, None, None], + ] + + +def test_scatter_flat_crossproduct_empty_arrays() -> None: + """Test that flat_dotproduct scattering empty arrays skips execution.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [] + + +def test_scatter_flat_crossproduct_and_conditional() -> None: + """Test flat_crossproduct scattering with partially empty arrays.""" + err_code, stdout, stderr = get_main_output( + [ + get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"), + get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"), + ] + ) + result = json.loads(stdout)["optional_echoed_messages"] + stderr = re.sub(r"\s\s+", " ", stderr) + assert err_code == 0, stderr + assert result == [ + "We Never\n", + "We Out\n", + "We Anything But\n", + "We -- The Aliens\n", + "Come Never\n", + "Come Out\n", + "Come Anything But\n", + "Come -- The Aliens\n", + "In Never\n", + "In Out\n", + "In Anything But\n", + "In -- The Aliens\n", + "Peace Never\n", + "Peace Out\n", + "Peace Anything But\n", + "Peace -- The Aliens\n", + ] diff --git a/tests/wf/scatter_before_when-flat_crossproduct.cwl b/tests/wf/scatter_before_when-flat_crossproduct.cwl new file mode 100644 index 000000000..afb3de55f --- /dev/null +++ b/tests/wf/scatter_before_when-flat_crossproduct.cwl @@ -0,0 +1,38 @@ +cwlVersion: v1.2 +class: Workflow + +requirements: + ScatterFeatureRequirement: {} + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} + +inputs: + messages: + type: + - "null" + - type: array + items: [string, "null"] + extras: + type: + - "null" + - type: array + items: [string, "null"] + +steps: + optional_echo_scatter: + when: $(inputs.messages !== null && inputs.extra !== null) + run: ../echo.cwl + scatter: [messages, extra] + scatterMethod: flat_crossproduct + in: + extra: extras + messages: messages + inp: + valueFrom: $(inputs.messages) $(inputs.extra) + out: [out] + +outputs: + optional_echoed_messages: + type: string[]? + pickValue: all_non_null + outputSource: optional_echo_scatter/out diff --git a/tests/wf/scatter_before_when-inp.yaml b/tests/wf/scatter_before_when-inp.yaml new file mode 100644 index 000000000..096d02cec --- /dev/null +++ b/tests/wf/scatter_before_when-inp.yaml @@ -0,0 +1,6 @@ +messages: + - We + - come + - in + - null + - peace diff --git a/tests/wf/scatter_before_when-inp_empty.yaml b/tests/wf/scatter_before_when-inp_empty.yaml new file mode 100644 index 000000000..2c42d3152 --- /dev/null +++ b/tests/wf/scatter_before_when-inp_empty.yaml @@ -0,0 +1,2 @@ +messages: [] + diff --git a/tests/wf/scatter_before_when-nested_crossproduct.cwl b/tests/wf/scatter_before_when-nested_crossproduct.cwl new file mode 100644 index 000000000..36735d336 --- /dev/null +++ b/tests/wf/scatter_before_when-nested_crossproduct.cwl @@ -0,0 +1,38 @@ +cwlVersion: v1.2 +class: Workflow + +requirements: + ScatterFeatureRequirement: {} + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} + +inputs: + messages: + type: + - "null" + - type: array + items: [string, "null"] + extras: + type: + - "null" + - type: array + items: [string, "null"] + +steps: + optional_echo_scatter: + when: $(inputs.messages !== null && inputs.extra !== null) + run: ../echo.cwl + scatter: [messages, extra] + scatterMethod: nested_crossproduct + in: + extra: extras + messages: messages + inp: + valueFrom: $(inputs.messages) $(inputs.extra) + out: [out] + +outputs: + optional_echoed_messages: + type: string[]? + pickValue: all_non_null + outputSource: optional_echo_scatter/out diff --git a/tests/wf/scatter_before_when.cwl b/tests/wf/scatter_before_when.cwl new file mode 100644 index 000000000..8eb15cbc7 --- /dev/null +++ b/tests/wf/scatter_before_when.cwl @@ -0,0 +1,28 @@ +cwlVersion: v1.2 +class: Workflow + +requirements: + ScatterFeatureRequirement: {} + InlineJavascriptRequirement: {} + +inputs: + messages: + type: + - "null" + - type: array + items: [string, "null"] + +steps: + optional_echo_scatter: + when: $(inputs.inp !== null) + run: ../echo.cwl + scatter: inp + in: + inp: messages + out: [out] + +outputs: + optional_echoed_messages: + type: string[]? + pickValue: all_non_null + outputSource: optional_echo_scatter/out diff --git a/tests/wf/scatter_before_when_dotproduct-inp.yaml b/tests/wf/scatter_before_when_dotproduct-inp.yaml new file mode 100644 index 000000000..91acc007a --- /dev/null +++ b/tests/wf/scatter_before_when_dotproduct-inp.yaml @@ -0,0 +1,12 @@ +messages: + - We + - Come + - In + - null + - Peace +extras: + - Never + - Out + - Anything But + - null + - "-- The Aliens" diff --git a/tests/wf/scatter_before_when_dotproduct.cwl b/tests/wf/scatter_before_when_dotproduct.cwl new file mode 100644 index 000000000..10cb535d7 --- /dev/null +++ b/tests/wf/scatter_before_when_dotproduct.cwl @@ -0,0 +1,38 @@ +cwlVersion: v1.2 +class: Workflow + +requirements: + ScatterFeatureRequirement: {} + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} + +inputs: + messages: + type: + - "null" + - type: array + items: [string, "null"] + extras: + type: + - "null" + - type: array + items: [string, "null"] + +steps: + optional_echo_scatter: + when: $(inputs.messages !== null && inputs.extras !== null) + run: ../echo.cwl + scatter: [messages, extra] + scatterMethod: dotproduct + in: + extra: extras + messages: messages + inp: + valueFrom: $(inputs.messages) $(inputs.extra) + out: [out] + +outputs: + optional_echoed_messages: + type: string[]? + pickValue: all_non_null + outputSource: optional_echo_scatter/out