From 8af88ce0d9d37b109582d02b6d91394acc4f1db5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 14:36:31 +0200 Subject: [PATCH 1/2] drop pack.py in favor of cwl_utils.pack Only 7 test failures! --- cwltool/main.py | 17 +-- cwltool/pack.py | 323 --------------------------------------------- setup.py | 1 - tests/test_pack.py | 97 ++------------ 4 files changed, 14 insertions(+), 424 deletions(-) delete mode 100644 cwltool/pack.py diff --git a/cwltool/main.py b/cwltool/main.py index 63e3d8679..9002b3dcf 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -37,6 +37,7 @@ import coloredlogs import pkg_resources # part of setuptools import ruamel.yaml +from cwl_utils.pack import pack from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -68,7 +69,6 @@ from .loghandler import _logger, configure_logging, defaultStreamHandler from .mpi import MpiConfig from .mutation import MutationManager -from .pack import pack from .process import ( CWL_IANA, Process, @@ -629,16 +629,11 @@ def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None] def print_pack( - loadingContext: LoadingContext, uri: str, ) -> str: """Return a CWL serialization of the CWL document in JSON.""" - packed = pack(loadingContext, uri) - if len(cast(Sized, packed["$graph"])) > 1: - return json_dumps(packed, indent=4, default=str) - return json_dumps( - cast(MutableSequence[CWLObjectType], packed["$graph"])[0], indent=4, default=str - ) + packed = pack(uri) + return json_dumps(packed, indent=4, default=str) def supported_cwl_versions(enable_dev: bool) -> List[str]: @@ -1127,14 +1122,12 @@ def main( processobj, metadata = loadingContext.loader.resolve_ref(uri) processobj = cast(Union[CommentedMap, CommentedSeq], processobj) if args.pack: - print(print_pack(loadingContext, uri), file=stdout) + print(print_pack(uri), file=stdout) return 0 if args.provenance and runtimeContext.research_obj: # Can't really be combined with args.pack at same time - runtimeContext.research_obj.packed_workflow( - print_pack(loadingContext, uri) - ) + runtimeContext.research_obj.packed_workflow(print_pack(uri)) if args.print_pre: print( diff --git a/cwltool/pack.py b/cwltool/pack.py deleted file mode 100644 index efd7aeada..000000000 --- a/cwltool/pack.py +++ /dev/null @@ -1,323 +0,0 @@ -"""Reformat a CWL document and all its references to be a single stream.""" - -import copy -import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) - -from ruamel.yaml.comments import CommentedMap, CommentedSeq -from schema_salad.ref_resolver import Loader, SubLoader -from schema_salad.utils import ResolveType - -from .context import LoadingContext -from .load_tool import fetch_document, resolve_and_validate_document -from .process import shortname, uniquename -from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION, update -from .utils import CWLObjectType, CWLOutputType - -LoadRefType = Callable[[Optional[str], str], ResolveType] - - -def find_run( - d: Union[CWLObjectType, ResolveType], - loadref: LoadRefType, - runs: Set[str], -) -> None: - if isinstance(d, MutableSequence): - for s in d: - find_run(s, loadref, runs) - elif isinstance(d, MutableMapping): - if "run" in d and isinstance(d["run"], str): - if d["run"] not in runs: - runs.add(d["run"]) - find_run(loadref(None, d["run"]), loadref, runs) - for s in d.values(): - find_run(s, loadref, runs) - - -def find_ids( - d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], -) -> None: - if isinstance(d, MutableSequence): - for s in d: - find_ids(cast(CWLObjectType, s), ids) - elif isinstance(d, MutableMapping): - for i in ("id", "name"): - if i in d and isinstance(d[i], str): - ids.add(cast(str, d[i])) - for s2 in d.values(): - find_ids(cast(CWLOutputType, s2), ids) - - -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: - if isinstance(d, MutableSequence): - for s, v in enumerate(d): - if isinstance(v, str): - if v in rewrite: - d[s] = rewrite[v] - elif v.startswith(stem): - d[s] = newstem + v[len(stem) :] - rewrite[v] = d[s] - else: - replace_refs(v, rewrite, stem, newstem) - elif isinstance(d, MutableMapping): - for key, val in d.items(): - if isinstance(val, str): - if val in rewrite: - d[key] = rewrite[val] - elif val.startswith(stem): - id_ = val[len(stem) :] - # prevent appending newstems if tool is already packed - if id_.startswith(newstem.strip("#")): - d[key] = "#" + id_ - else: - d[key] = newstem + id_ - rewrite[val] = d[key] - replace_refs(val, rewrite, stem, newstem) - - -def import_embed( - d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], -) -> None: - if isinstance(d, MutableSequence): - for v in d: - import_embed(cast(CWLOutputType, v), seen) - elif isinstance(d, MutableMapping): - for n in ("id", "name"): - if n in d: - if isinstance(d[n], str): - ident = cast(str, d[n]) - if ident in seen: - this = ident - d.clear() - d["$import"] = this - else: - this = ident - seen.add(this) - break - - for k in sorted(d.keys()): - import_embed(cast(CWLOutputType, d[k]), seen) - - -def pack( - loadingContext: LoadingContext, - uri: str, - rewrite_out: Optional[Dict[str, str]] = None, - loader: Optional[Loader] = None, -) -> CWLObjectType: - - # The workflow document we have in memory right now may have been - # updated to the internal CWL version. We need to reload the - # document to go back to its original version. - # - # What's going on here is that the updater replaces the - # documents/fragments in the index with updated ones, the - # index is also used as a cache, so we need to go through the - # loading process with an empty index and updating turned off - # so we have the original un-updated documents. - # - loadingContext = loadingContext.copy() - document_loader = SubLoader(loader or loadingContext.loader or Loader({})) - loadingContext.do_update = False - loadingContext.loader = document_loader - loadingContext.loader.idx = {} - loadingContext.metadata = {} - loadingContext, docobj, uri = fetch_document(uri, loadingContext) - loadingContext, fileuri = resolve_and_validate_document( - loadingContext, docobj, uri, preprocess_only=True - ) - if loadingContext.loader is None: - raise Exception("loadingContext.loader cannot be none") - processobj, metadata = loadingContext.loader.resolve_ref(uri) - document_loader = loadingContext.loader - - if isinstance(processobj, MutableMapping): - document_loader.idx[processobj["id"]] = CommentedMap(processobj.items()) - elif isinstance(processobj, MutableSequence): - _, frag = urllib.parse.urldefrag(uri) - for po in processobj: - if not frag: - if po["id"].endswith("#main"): - uri = po["id"] - document_loader.idx[po["id"]] = CommentedMap(po.items()) - document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - - found_versions = { - cast(str, loadingContext.metadata["cwlVersion"]) - } # type: Set[str] - - def loadref(base: Optional[str], lr_uri: str) -> ResolveType: - lr_loadingContext = loadingContext.copy() - lr_loadingContext.metadata = {} - lr_loadingContext, lr_workflowobj, lr_uri = fetch_document( - lr_uri, lr_loadingContext - ) - lr_loadingContext, lr_uri = resolve_and_validate_document( - lr_loadingContext, lr_workflowobj, lr_uri - ) - found_versions.add(cast(str, lr_loadingContext.metadata["cwlVersion"])) - if lr_loadingContext.loader is None: - raise Exception("loader should not be None") - return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - - input_ids: Set[str] = set() - output_ids: Set[str] = set() - - if isinstance(processobj, MutableSequence): - mainobj = processobj[0] - else: - mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) - - runs = {uri} - find_run(processobj, loadref, runs) - - # Figure out the highest version, everything needs to be updated - # to it. - m = 0 - for fv in found_versions: - m = max(m, ORDERED_VERSIONS.index(fv)) - update_to_version = ORDERED_VERSIONS[m] - - for f in runs: - find_ids(document_loader.resolve_ref(f)[0], input_ids) - - input_names: Set[str] = set() - output_names: Set[str] = set() - - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} - - mainpath, _ = urllib.parse.urldefrag(uri) - - def rewrite_id( - r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str] - ) -> None: - if r == mainuri: - rewrite[r] = "#main" - elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): - if r[len(mainuri) :].startswith("#main/"): - rewrite[r] = "#" + uniquename(r[len(mainuri) + 1 :], names) - else: - rewrite[r] = "#" + uniquename("main/" + r[len(mainuri) + 1 :], names) - else: - path, frag = urllib.parse.urldefrag(r) - if path == mainpath: - rewrite[r] = "#" + uniquename(frag, names) - else: - if path not in rewrite: - rewrite[path] = "#" + uniquename(shortname(path), names) - - sorted_input_ids = sorted(input_ids) - sorted_output_ids = sorted(output_ids) - - for r in sorted_input_ids: - rewrite_id(r, uri, rewrite_inputs, input_names) - for r in sorted_output_ids: - rewrite_id(r, uri, rewrite_outputs, output_names) - - packed = CommentedMap( - (("$graph", CommentedSeq()), ("cwlVersion", update_to_version)) - ) - namespaces = metadata.get("$namespaces", None) - - schemas: Set[str] = set() - if "$schemas" in metadata: - for each_schema in metadata["$schemas"]: - schemas.add(each_schema) - for r in sorted(runs): - dcr, metadata = document_loader.resolve_ref(r) - if isinstance(dcr, CommentedSeq): - dcr = dcr[0] - dcr = cast(CommentedMap, dcr) - if not isinstance(dcr, MutableMapping): - continue - - dcr = update( - dcr, - document_loader, - r, - loadingContext.enable_dev, - metadata, - update_to_version, - ) - - if ORIGINAL_CWLVERSION in metadata: - del metadata[ORIGINAL_CWLVERSION] - if ORIGINAL_CWLVERSION in dcr: - del dcr[ORIGINAL_CWLVERSION] - - if "$schemas" in metadata: - for s in metadata["$schemas"]: - schemas.add(s) - if dcr.get("class") not in ( - "Workflow", - "CommandLineTool", - "ExpressionTool", - "Operation", - ): - continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) - v = rewrite_inputs[r] - dc["id"] = v - for n in ("name", "cwlVersion", "$namespaces", "$schemas"): - if n in dc: - del dc[n] - packed["$graph"].append(dc) - - if schemas: - packed["$schemas"] = list(schemas) - if namespaces: - packed["$namespaces"] = namespaces - - save_outputs = packed["$graph"][0].pop("outputs") - for r in list(rewrite_inputs.keys()): - v = rewrite_inputs[r] - replace_refs(packed, rewrite_inputs, r + "/" if "#" in r else r + "#", v + "/") - - import_embed(packed, set()) - - packed["$graph"][0]["outputs"] = save_outputs - for r in list(rewrite_outputs.keys()): - v = rewrite_outputs[r] - replace_refs( - packed["$graph"][0]["outputs"], - rewrite_outputs, - r + "/" if "#" in r else r + "#", - v + "/", - ) - - for r in list( - rewrite_inputs.keys() - ): # again, to process the outputSource references - v = rewrite_inputs[r] - replace_refs(packed, rewrite_inputs, r + "/" if "#" in r else r + "#", v + "/") - - if len(packed["$graph"]) == 1: - # duplicate 'cwlVersion', '$schemas', and '$namespaces' inside '$graph' - # when there is only a single item because main.print_pack() will print - # the contents inside '$graph' rather than whole dict in this case - packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"] - if schemas: - packed["$graph"][0]["$schemas"] = list(schemas) - if namespaces: - packed["$graph"][0]["$namespaces"] = namespaces - - if rewrite_out is not None: - rewrite_out.update(rewrite_inputs) - rewrite_out.update(rewrite_outputs) - - return packed diff --git a/setup.py b/setup.py index ddac1aafb..7a3eef372 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ # "cwltool/__main__.py", "cwltool/main.py", "cwltool/mutation.py", - "cwltool/pack.py", # "cwltool/pathmapper.py", # class PathMapper needs to be subclassable "cwltool/process.py", "cwltool/procgenerator.py", diff --git a/tests/test_pack.py b/tests/test_pack.py index 40797b8d8..bd333e792 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -8,9 +8,9 @@ from typing import Dict import pytest +from cwl_utils.pack import pack from schema_salad.utils import yaml_no_ts -import cwltool.pack import cwltool.workflow from cwltool.context import LoadingContext from cwltool.load_tool import fetch_document, resolve_and_validate_document @@ -42,13 +42,7 @@ ) def test_packing(unpacked: str, expected: str) -> None: """Compare expected version reality with various workflows and --pack.""" - loadingContext, workflowobj, uri = fetch_document(get_data(unpacked)) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - - packed = json.loads(print_pack(loadingContext, uri)) + packed = json.loads(print_pack(get_data(unpacked))) context_dir = os.path.abspath(os.path.dirname(get_data(unpacked))) adjustFileObjs(packed, partial(make_relative, context_dir)) adjustDirObjs(packed, partial(make_relative, context_dir)) @@ -68,18 +62,7 @@ def test_packing(unpacked: str, expected: str) -> None: def test_pack_single_tool() -> None: - loadingContext, workflowobj, uri = fetch_document( - get_data("tests/wf/formattest.cwl") - ) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - - packed = cwltool.pack.pack(loadingContext, uri) + packed = pack(get_data("tests/wf/formattest.cwl")) assert "$schemas" in packed @@ -90,7 +73,7 @@ def test_pack_fragment() -> None: expect_packed = yaml.load(packed_file) loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/scatter2.cwl")) - packed = cwltool.pack.pack(loadingContext, uri + "#scatterstep/mysub") + packed = pack(uri + "#scatterstep/mysub") adjustFileObjs( packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))) ) @@ -102,29 +85,6 @@ def test_pack_fragment() -> None: assert packed_result == expected -def test_pack_rewrites() -> None: - rewrites = {} # type: Dict[str, str] - - loadingContext, workflowobj, uri = fetch_document( - get_data("tests/wf/default-wf5.cwl") - ) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - - cwltool.pack.pack( - loadingContext, - uri, - rewrite_out=rewrites, - ) - - assert len(rewrites) == 6 - - cwl_missing_version_paths = [ "tests/wf/hello_single_tool.cwl", "tests/wf/hello-workflow.cwl", @@ -135,17 +95,7 @@ def test_pack_rewrites() -> None: def test_pack_missing_cwlVersion(cwl_path: str) -> None: """Ensure the generated pack output is not missing the `cwlVersion` in case of single tool workflow and single step workflow.""" # Testing single tool workflow - loadingContext, workflowobj, uri = fetch_document(get_data(cwl_path)) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - - # generate pack output dict - packed = json.loads(print_pack(loadingContext, uri)) + packed = json.loads(print_pack(get_data(cwl_path))) assert packed["cwlVersion"] == "v1.0" @@ -161,17 +111,8 @@ def test_pack_idempotence_workflow(tmp_path: Path) -> None: def _pack_idempotently(document: str, tmp_path: Path) -> None: - loadingContext, workflowobj, uri = fetch_document(get_data(document)) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - # generate pack output dict - packed_text = print_pack(loadingContext, uri) + packed_text = print_pack(get_data(document)) packed = json.loads(packed_text) tmp_name = tmp_path / "packed.cwl" @@ -180,20 +121,10 @@ def _pack_idempotently(document: str, tmp_path: Path) -> None: tmp.flush() tmp.close() - loadingContext, workflowobj, uri2 = fetch_document(tmp.name) - loadingContext.do_update = False - loadingContext, uri2 = resolve_and_validate_document( - loadingContext, workflowobj, uri2 - ) - loader2 = loadingContext.loader - assert loader2 - loader2.resolve_ref(uri2)[0] - - # generate pack output dict - packed_text = print_pack(loadingContext, uri2) + # generate 2nd packed output dict + packed_text = print_pack(tmp.name) double_packed = json.loads(packed_text) - assert uri != uri2 assert packed == double_packed @@ -208,17 +139,7 @@ def _pack_idempotently(document: str, tmp_path: Path) -> None: def test_packed_workflow_execution( wf_path: str, job_path: str, namespaced: bool, tmp_path: Path ) -> None: - loadingContext = LoadingContext() - loadingContext.resolver = tool_resolver - loadingContext, workflowobj, uri = fetch_document(get_data(wf_path), loadingContext) - loadingContext.do_update = False - loadingContext, uri = resolve_and_validate_document( - loadingContext, workflowobj, uri - ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - packed = json.loads(print_pack(loadingContext, uri)) + packed = json.loads(print_pack(get_data(wf_path))) assert not namespaced or "$namespaces" in packed From cbc0a692822d894f7fc17df294cf43581decabb5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 14:57:22 +0200 Subject: [PATCH 2/2] sync the changes in packing format --- tests/wf/expect_iwd-passthrough1_packed.cwl | 11 +- tests/wf/expect_packed.cwl | 228 ++++++++------- tests/wf/expect_revsort_datetime_packed.cwl | 234 ++++++++-------- .../expect_operation-single_packed.cwl | 260 +++++++++--------- 4 files changed, 362 insertions(+), 371 deletions(-) diff --git a/tests/wf/expect_iwd-passthrough1_packed.cwl b/tests/wf/expect_iwd-passthrough1_packed.cwl index 86db3734b..d0b44c6cd 100644 --- a/tests/wf/expect_iwd-passthrough1_packed.cwl +++ b/tests/wf/expect_iwd-passthrough1_packed.cwl @@ -1,5 +1,6 @@ { "class": "CommandLineTool", + "cwlVersion": "v1.2", "doc": "YAML |- syntax does not add trailing newline so in the listing entry\nbelow there is no whitespace surrounding the value\n$(inputs.filelist), so it is evaluated as a File object. Compare to\niwd-passthrough2.cwl\n", "requirements": [ { @@ -15,19 +16,17 @@ "inputs": [ { "type": "File", - "id": "#main/filelist" + "id": "filelist" } ], - "baseCommand": "true", - "id": "#main", "outputs": [ { "type": "File", "outputBinding": { "glob": "renamed-filelist.txt" }, - "id": "#main/filelist" + "id": "filelist" } ], - "cwlVersion": "v1.2" -} \ No newline at end of file + "baseCommand": "true" +} diff --git a/tests/wf/expect_packed.cwl b/tests/wf/expect_packed.cwl index ecd11dfb2..c0e16571c 100644 --- a/tests/wf/expect_packed.cwl +++ b/tests/wf/expect_packed.cwl @@ -1,137 +1,135 @@ { - "$graph": [ + "class": "Workflow", + "doc": "Reverse the lines in a document, then sort those lines.", + "cwlVersion": "v1.0", + "hints": [ { - "class": "Workflow", - "doc": "Reverse the lines in a document, then sort those lines.", - "hints": [ - { - "class": "DockerRequirement", - "dockerPull": "docker.io/debian:stable-slim" - } - ], - "inputs": [ - { - "type": "boolean", - "default": true, - "doc": "If true, reverse (descending) sort", - "id": "#main/reverse_sort" - }, - { - "type": "File", - "doc": "The input file to be processed.", - "format": "https://www.iana.org/assignments/media-types/text/plain", - "default": { - "class": "File", - "location": "hello.txt" - }, - "id": "#main/workflow_input" - } - ], - "outputs": [ - { - "type": "File", - "outputSource": "#main/sorted/sorted_output", - "doc": "The output with the lines reversed and sorted.", - "id": "#main/sorted_output" - } - ], - "steps": [ - { - "in": [ - { - "source": "#main/workflow_input", - "id": "#main/rev/revtool_input" - } - ], - "out": [ - "#main/rev/revtool_output" - ], - "run": "#revtool.cwl", - "id": "#main/rev" - }, - { - "in": [ - { - "source": "#main/reverse_sort", - "id": "#main/sorted/reverse" - }, - { - "source": "#main/rev/revtool_output", - "id": "#main/sorted/sorted_input" - } - ], - "out": [ - "#main/sorted/sorted_output" - ], - "run": "#sorttool.cwl", - "id": "#main/sorted" - } - ], - "id": "#main" + "class": "DockerRequirement", + "dockerPull": "docker.io/debian:stable-slim" + } + ], + "inputs": [ + { + "type": "File", + "doc": "The input file to be processed.", + "format": "iana:text/plain", + "default": { + "class": "File", + "location": "hello.txt" + }, + "id": "workflow_input" }, { - "class": "CommandLineTool", - "doc": "Reverse each line using the `rev` command", - "inputs": [ + "type": "boolean", + "default": true, + "doc": "If true, reverse (descending) sort", + "id": "reverse_sort" + } + ], + "outputs": [ + { + "type": "File", + "outputSource": "sorted/sorted_output", + "doc": "The output with the lines reversed and sorted.", + "id": "sorted_output" + } + ], + "steps": [ + { + "in": [ { - "type": "File", - "inputBinding": {}, - "id": "#revtool.cwl/revtool_input" + "source": "workflow_input", + "id": "revtool_input" } ], - "outputs": [ - { - "type": "File", - "outputBinding": { - "glob": "output.txt" - }, - "id": "#revtool.cwl/revtool_output" - } + "out": [ + "revtool_output" ], - "baseCommand": "rev", - "stdout": "output.txt", - "id": "#revtool.cwl" + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "doc": "Reverse each line using the `rev` command", + "$schemas": [ + "empty.ttl" + ], + "inputs": [ + { + "type": "File", + "inputBinding": {}, + "id": "revtool_input" + } + ], + "outputs": [ + { + "type": "File", + "outputBinding": { + "glob": "output.txt" + }, + "id": "revtool_output" + } + ], + "baseCommand": "rev", + "stdout": "output.txt", + "requirements": [] + }, + "id": "rev" }, { - "class": "CommandLineTool", - "doc": "Sort lines using the `sort` command", - "inputs": [ + "in": [ { - "id": "#sorttool.cwl/reverse", - "type": "boolean", - "inputBinding": { - "position": 1, - "prefix": "--reverse" - } + "source": "rev/revtool_output", + "id": "sorted_input" }, { - "id": "#sorttool.cwl/sorted_input", - "type": "File", - "inputBinding": { - "position": 2 - } + "source": "reverse_sort", + "id": "reverse" } ], - "outputs": [ - { - "id": "#sorttool.cwl/sorted_output", - "type": "File", - "outputBinding": { - "glob": "output.txt" - } - } + "out": [ + "sorted_output" ], - "baseCommand": "sort", - "stdout": "output.txt", - "id": "#sorttool.cwl" + "run": { + "class": "CommandLineTool", + "doc": "Sort lines using the `sort` command", + "cwlVersion": "v1.0", + "inputs": [ + { + "id": "reverse", + "type": "boolean", + "inputBinding": { + "position": 1, + "prefix": "--reverse" + } + }, + { + "id": "sorted_input", + "type": "File", + "inputBinding": { + "position": 2 + } + } + ], + "outputs": [ + { + "id": "sorted_output", + "type": "File", + "outputBinding": { + "glob": "output.txt" + } + } + ], + "baseCommand": "sort", + "stdout": "output.txt", + "requirements": [] + }, + "id": "sorted" } ], - "cwlVersion": "v1.0", + "$namespaces": { + "iana": "https://www.iana.org/assignments/media-types/" + }, "$schemas": [ - "empty.ttl", "empty2.ttl" ], - "$namespaces": { - "iana": "https://www.iana.org/assignments/media-types/" - } + "requirements": [] } diff --git a/tests/wf/expect_revsort_datetime_packed.cwl b/tests/wf/expect_revsort_datetime_packed.cwl index c9a84fa45..175f92d57 100644 --- a/tests/wf/expect_revsort_datetime_packed.cwl +++ b/tests/wf/expect_revsort_datetime_packed.cwl @@ -1,140 +1,138 @@ { - "$graph": [ + "class": "Workflow", + "doc": "Reverse the lines in a document, then sort those lines.", + "cwlVersion": "v1.0", + "hints": [ { - "class": "Workflow", - "doc": "Reverse the lines in a document, then sort those lines.", - "hints": [ - { - "class": "DockerRequirement", - "dockerPull": "docker.io/debian:stable-slim" - } - ], - "inputs": [ - { - "type": "boolean", - "default": true, - "doc": "If true, reverse (descending) sort", - "id": "#main/reverse_sort" - }, - { - "type": "File", - "doc": "The input file to be processed.", - "format": "https://www.iana.org/assignments/media-types/text/plain", - "default": { - "class": "File", - "location": "hello.txt" - }, - "id": "#main/workflow_input" - } - ], - "outputs": [ - { - "type": "File", - "outputSource": "#main/sorted/sorted_output", - "doc": "The output with the lines reversed and sorted.", - "id": "#main/sorted_output" - } - ], - "steps": [ - { - "in": [ - { - "source": "#main/workflow_input", - "id": "#main/rev/revtool_input" - } - ], - "out": [ - "#main/rev/revtool_output" - ], - "run": "#revtool.cwl", - "id": "#main/rev" - }, - { - "in": [ - { - "source": "#main/reverse_sort", - "id": "#main/sorted/reverse" - }, - { - "source": "#main/rev/revtool_output", - "id": "#main/sorted/sorted_input" - } - ], - "out": [ - "#main/sorted/sorted_output" - ], - "run": "#sorttool.cwl", - "id": "#main/sorted" - } - ], - "id": "#main", - "http://schema.org/dateCreated": "2020-10-08" + "class": "DockerRequirement", + "dockerPull": "docker.io/debian:stable-slim" + } + ], + "inputs": [ + { + "type": "File", + "doc": "The input file to be processed.", + "format": "iana:text/plain", + "default": { + "class": "File", + "location": "hello.txt" + }, + "id": "workflow_input" }, { - "class": "CommandLineTool", - "doc": "Reverse each line using the `rev` command", - "inputs": [ + "type": "boolean", + "default": true, + "doc": "If true, reverse (descending) sort", + "id": "reverse_sort" + } + ], + "outputs": [ + { + "type": "File", + "outputSource": "sorted/sorted_output", + "doc": "The output with the lines reversed and sorted.", + "id": "sorted_output" + } + ], + "steps": [ + { + "in": [ { - "type": "File", - "inputBinding": {}, - "id": "#revtool.cwl/revtool_input" + "source": "workflow_input", + "id": "revtool_input" } ], - "outputs": [ - { - "type": "File", - "outputBinding": { - "glob": "output.txt" - }, - "id": "#revtool.cwl/revtool_output" - } + "out": [ + "revtool_output" ], - "baseCommand": "rev", - "stdout": "output.txt", - "id": "#revtool.cwl" + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "doc": "Reverse each line using the `rev` command", + "$schemas": [ + "empty.ttl" + ], + "inputs": [ + { + "type": "File", + "inputBinding": {}, + "id": "revtool_input" + } + ], + "outputs": [ + { + "type": "File", + "outputBinding": { + "glob": "output.txt" + }, + "id": "revtool_output" + } + ], + "baseCommand": "rev", + "stdout": "output.txt", + "requirements": [] + }, + "id": "rev" }, { - "class": "CommandLineTool", - "doc": "Sort lines using the `sort` command", - "inputs": [ + "in": [ { - "id": "#sorttool.cwl/reverse", - "type": "boolean", - "inputBinding": { - "position": 1, - "prefix": "--reverse" - } + "source": "rev/revtool_output", + "id": "sorted_input" }, { - "id": "#sorttool.cwl/sorted_input", - "type": "File", - "inputBinding": { - "position": 2 - } + "source": "reverse_sort", + "id": "reverse" } ], - "outputs": [ - { - "id": "#sorttool.cwl/sorted_output", - "type": "File", - "outputBinding": { - "glob": "output.txt" - } - } + "out": [ + "sorted_output" ], - "baseCommand": "sort", - "stdout": "output.txt", - "id": "#sorttool.cwl" + "run": { + "class": "CommandLineTool", + "doc": "Sort lines using the `sort` command", + "cwlVersion": "v1.0", + "inputs": [ + { + "id": "reverse", + "type": "boolean", + "inputBinding": { + "position": 1, + "prefix": "--reverse" + } + }, + { + "id": "sorted_input", + "type": "File", + "inputBinding": { + "position": 2 + } + } + ], + "outputs": [ + { + "id": "sorted_output", + "type": "File", + "outputBinding": { + "glob": "output.txt" + } + } + ], + "baseCommand": "sort", + "stdout": "output.txt", + "requirements": [] + }, + "id": "sorted" } ], - "cwlVersion": "v1.0", - "$schemas": [ - "empty2.ttl", - "https://schema.org/version/latest/schemaorg-current-https.rdf", - "empty.ttl" - ], "$namespaces": { "iana": "https://www.iana.org/assignments/media-types/", "s": "http://schema.org/" - } + }, + "$schemas": [ + "https://schema.org/version/latest/schemaorg-current-https.rdf", + "empty2.ttl" + ], + "s:dateCreated": "2020-10-08", + "requirements": [] } diff --git a/tests/wf/operation/expect_operation-single_packed.cwl b/tests/wf/operation/expect_operation-single_packed.cwl index 7e0782c40..f7db26963 100644 --- a/tests/wf/operation/expect_operation-single_packed.cwl +++ b/tests/wf/operation/expect_operation-single_packed.cwl @@ -1,154 +1,150 @@ { - "$graph": [ + "class": "Workflow", + "cwlVersion": "v1.2", + "id": "abstract_cosifer_workflow", + "label": "abstract-cosifer-workflow", + "inputs": [ { - "class": "Operation", - "requirements": [ - { - "dockerPull": "docker.io/tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e", - "class": "DockerRequirement" - } + "type": "File", + "doc": "Gene expression data matrix", + "id": "data_matrix" + }, + { + "type": [ + "null", + "File" ], - "inputs": [ - { - "type": "File", - "id": "#abstract-cosifer.cwl/data_matrix" - }, - { - "type": [ - "null", - "File" - ], - "id": "#abstract-cosifer.cwl/gmt_filepath" - }, - { - "type": [ - "null", - "int" - ], - "id": "#abstract-cosifer.cwl/index_col" - }, - { - "type": [ - "null", - "string" - ], - "id": "#abstract-cosifer.cwl/outdir" - }, - { - "type": [ - "null", - "boolean" - ], - "id": "#abstract-cosifer.cwl/samples_on_rows" - }, - { - "type": [ - "null", - "string" - ], - "doc": "The separator used in the data_matrix file", - "id": "#abstract-cosifer.cwl/separator" - } + "doc": "Optional GMT file to perform inference on multiple gene sets", + "id": "gmt_filepath" + }, + { + "type": [ + "null", + "int" ], - "id": "#abstract-cosifer.cwl", - "outputs": [ - { - "type": "Directory", - "id": "#abstract-cosifer.cwl/resdir" - } - ] + "doc": "Column index in the data. Defaults to None, a.k.a., no index", + "id": "index_col" }, { - "class": "Workflow", - "id": "#main", - "label": "abstract-cosifer-workflow", - "inputs": [ - { - "type": "File", - "doc": "Gene expression data matrix", - "id": "#main/data_matrix" + "type": "string", + "doc": "Path to the output directory", + "id": "outdir" + }, + { + "type": [ + "null", + "string" + ], + "doc": "Separator for the data. Defaults to .", + "id": "separator" + }, + { + "type": [ + "null", + "boolean" + ], + "doc": "Flag that indicates that data contain the samples on rows. Defaults to False.", + "id": "samples_on_rows" + } + ], + "outputs": [ + { + "type": "Directory", + "outputSource": "abstract_cosifer/resdir", + "id": "resdir" + } + ], + "steps": [ + { + "run": { + "class": "Operation", + "cwlVersion": "v1.2", + "requirements": [ + { + "dockerPull": "docker.io/tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e", + "class": "DockerRequirement" + } + ], + "inputs": [ + { + "type": "File", + "id": "data_matrix" + }, + { + "type": [ + "null", + "string" + ], + "doc": "The separator used in the data_matrix file", + "id": "separator" + }, + { + "type": [ + "null", + "int" + ], + "id": "index_col" + }, + { + "type": [ + "null", + "File" + ], + "id": "gmt_filepath" + }, + { + "type": [ + "null", + "string" + ], + "id": "outdir" + }, + { + "type": [ + "null", + "boolean" + ], + "id": "samples_on_rows" + } + ], + "outputs": [ + { + "type": "Directory", + "id": "resdir" + } + ] + }, + "in": [ + { + "source": "data_matrix", + "id": "data_matrix" }, { - "type": [ - "null", - "File" - ], - "doc": "Optional GMT file to perform inference on multiple gene sets", - "id": "#main/gmt_filepath" + "source": "separator", + "id": "separator" }, { - "type": [ - "null", - "int" - ], - "doc": "Column index in the data. Defaults to None, a.k.a., no index", - "id": "#main/index_col" + "source": "index_col", + "id": "index_col" }, { - "type": "string", - "doc": "Path to the output directory", - "id": "#main/outdir" + "source": "gmt_filepath", + "id": "gmt_filepath" }, { - "type": [ - "null", - "boolean" - ], - "doc": "Flag that indicates that data contain the samples on rows. Defaults to False.", - "id": "#main/samples_on_rows" + "source": "outdir", + "id": "outdir" }, { - "type": [ - "null", - "string" - ], - "doc": "Separator for the data. Defaults to .", - "id": "#main/separator" + "source": "samples_on_rows", + "id": "samples_on_rows" } ], - "outputs": [ - { - "type": "Directory", - "outputSource": "#main/abstract_cosifer/resdir", - "id": "#main/resdir" - } + "out": [ + "resdir" ], - "steps": [ - { - "run": "#abstract-cosifer.cwl", - "in": [ - { - "source": "#main/data_matrix", - "id": "#main/abstract_cosifer/data_matrix" - }, - { - "source": "#main/gmt_filepath", - "id": "#main/abstract_cosifer/gmt_filepath" - }, - { - "source": "#main/index_col", - "id": "#main/abstract_cosifer/index_col" - }, - { - "source": "#main/outdir", - "id": "#main/abstract_cosifer/outdir" - }, - { - "source": "#main/samples_on_rows", - "id": "#main/abstract_cosifer/samples_on_rows" - }, - { - "source": "#main/separator", - "id": "#main/abstract_cosifer/separator" - } - ], - "out": [ - "#main/abstract_cosifer/resdir" - ], - "id": "#main/abstract_cosifer" - } - ] + "id": "abstract_cosifer" } ], - "cwlVersion": "v1.2" + "requirements": [] }