Allow $imports in type fields

Brings in rabix/sbpack#67
+ the packing tests that we didn't originally include when we brought
in sbpack to cwl-utils

Co-authored-by: Thomas Hickman <[email protected]>

MANIFEST.in

@@ -13,6 +13,11 @@ include testdata/*.yaml
 include testdata/*.input
 include testdata/*.ttl
 include testdata/*.owl
+include testdata/*.js
+include testdata/remote-cwl/*.cwl
+include testdata/workflows/*.cwl
+include testdata/workflows/*.yaml
+include testdata/types/*.yml
 include testdata/checker_wf/*.cwl
 include cwl_utils/py.typed
 include docs/conf.py docs/Makefile docs/_static/favicon.ico docs/requirements.txt

cwl_utils/schemadef.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
+#  Copyright (c) 2023 Genomics plc
 #  Copyright (c) 2021 Michael R. Crusoe
 #  Copyright (c) 2020 Seven Bridges
 #  See https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/LICENSE
@@ -191,6 +192,12 @@ def _inline_type(
         return [_inline_type(_v, base_url, user_defined_types) for _v in v]
 
     elif isinstance(v, dict):
+        if v.get("$import") is not None:
+            imported_type, import_base_url = utils.load_linked_file(
+                base_url, v["$import"], is_import=True
+            )
+            return _inline_type(imported_type, import_base_url, user_defined_types)
+
         _type = v.get("type")
         if _type is None:
             raise errors.MissingTypeName(

testdata/lib.js

@@ -0,0 +1,7 @@
+var foo = function(x) {
+    return 2 * x
+}
+
+var bar = function(n, x) {
+    return `{n} engineers walk into a {x}`
+}

testdata/remote-cwl/tool1.cwl

@@ -0,0 +1,26 @@
+# We have this tool to test both local and remote packing
+
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+  in1: 
+    type: string
+    inputBinding: 
+      position: 1
+      valueFrom: A_$(inputs.in1)_B_${return inputs.in1}_C_$(inputs.in1)
+baseCommand: echo
+arguments:
+  - valueFrom: $(runtime)
+outputs:
+  out1:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self)_D_$(runtime)
+
+stdout: out.txt
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - $include: ../lib.js

testdata/remote-cwl/tool2.cwl

@@ -0,0 +1,26 @@
+# We have this tool to test both local and remote packing
+
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+  in1:
+    type: ../types/testtypes.yml#my_boolean_array
+    inputBinding:
+      position: 1
+      valueFrom: A_$(inputs.in1)_B_${return inputs.in1}_C_$(inputs.in1)
+baseCommand: echo
+arguments:
+  - valueFrom: $(runtime)
+outputs:
+  out1:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self)_D_$(runtime)
+
+stdout: out.txt
+requirements:
+  SchemaDefRequirement:
+    types: 
+      - $import: ../types/testtypes.yml

testdata/remote-cwl/wf1.cwl

@@ -0,0 +1,27 @@
+class: Workflow
+cwlVersion: v1.0
+inputs:
+  - id: in1
+    type: ../types/testtypes.yml#my_boolean_array
+
+steps:
+  s1:
+    run: ./tool2.cwl
+    in:
+      in1: "#in1"   # This should be normalized out
+    out: [out1]
+  s2:
+    run: tool1.cwl
+    in:
+      in1: s1/out1
+    out: [out1]
+
+outputs:
+  - id: out1 
+    type: string 
+    outputSource: "#s2/out1"
+
+requirements:
+  SchemaDefRequirement:
+    types:
+      - $import: ../types/testtypes.yml

testdata/types/array.yml

@@ -0,0 +1,30 @@
+# Contains arrays and intra-file type reference
+
+# class: SchemaDefRequirement
+# types: ...
+# This form does not work with cwltool, even though it can be found here
+# https://github.com/NCI-GDC/gdc-dnaseq-cwl/blob/master/tools/readgroup_path.yml
+# Notably, that .yml file is never used, so likely they tried it, failed and
+# forgot to take it out
+
+
+  - name: sample_meta2  #duplicate names are not fine across files
+    type: record
+    fields:
+      - name: prop
+        type: string
+
+  - name: study_meta
+    type: array
+    items: sample_meta2
+
+  # Apparently can't declare an array inside an array?
+  # - name: study_meta_too
+  #   type: array
+  #   items: [string, sample_meta2, study_meta]
+
+  - name: study_meta_too
+    type: record
+    fields:
+      meta1: sample_meta2
+      meta2: study_meta

testdata/types/recursive.yml

@@ -0,0 +1,26 @@
+# Contains records and intra-file type reference
+# You can't have an enum in a user type
+
+- name: sample_meta
+  type: record
+  fields:
+    - name: sample
+      type: ["null", string]
+    - name: species
+      type: string
+
+- name: file_with_sample_meta
+  type: record
+  fields:
+    - name: file
+      type: File
+    - name: meta
+      type: sample_meta
+
+- name: info_with_sample_meta
+  type: record
+  fields:
+    comment: 
+      type: string
+    meta: 
+      type: sample_meta

testdata/types/singletype.yml

@@ -0,0 +1,7 @@
+# You can not use the dictionary format shown in
+# singletype2.yml. It has to be this
+
+name: simple_record
+type: record
+fields:
+  prop: string

testdata/types/singletype2.yml

@@ -0,0 +1,6 @@
+# You can not use this dictionary format
+
+simple_record2:
+  type: record
+  fields:
+    prop: string

testdata/types/testtypes.yml

@@ -0,0 +1,9 @@
+- name: my_boolean_array
+  type: array
+  items: boolean
+  label: "A boolean array"
+
+- name: my_enum
+  type: enum
+  symbols: [a, b, c]
+  label: "A required enum"

testdata/wf2.cwl

@@ -0,0 +1,29 @@
+class: Workflow
+cwlVersion: v1.0
+inputs:
+  in1: types/testtypes.yml#my_boolean_array
+  in2: 
+    type: types/testtypes.yml#my_enum
+
+steps:
+  s1:
+    run: remote-cwl/wf1.cwl
+    in:
+      - id: in1
+        source: "#in1"
+    out: [out1]
+  s2:
+    run: remote-cwl/tool1.cwl
+    in:
+      in1: s1/out1
+    out: [out1]
+
+outputs:
+  out1: 
+    type: string 
+    outputSource: "#s2/out1"
+
+requirements:
+  SchemaDefRequirement:
+    types:
+      - $import: types/testtypes.yml

testdata/workflows/count-lines16-wf.cwl

@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.2
+inputs:
+  file1: File
+outputs:
+  count_output: {type: int, outputSource: step1/count_output}
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    in: {file1: file1}
+    out: [count_output]
+    run:
+      class: Workflow
+      inputs:
+        file1: File
+      outputs:
+        count_output: {type: int, outputSource: step2/count_output}
+      steps:
+        step1: {run: wc-tool.cwl, in: {file1: file1}, out: [output]}
+        step2:
+          in: {file1: step1/output}
+          out: [count_output]
+          run:
+            class: Workflow
+            inputs:
+              file1: File
+            outputs:
+              count_output: {type: int, outputSource: step1/output}
+            steps:
+              step1: {run: parseInt-tool.cwl, in: {file1: file1}, out: [output]}

testdata/workflows/import-in-type.cwl

@@ -0,0 +1,19 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.2
+
+inputs:
+ file1:
+  type:
+    $import: type-import.yaml
+
+outputs:
+  output:
+    type: File
+    outputBinding: { glob: output }
+
+baseCommand: [wc, -l]
+
+stdin: $(inputs.file1.path)
+stdout: output

testdata/workflows/parseInt-tool.cwl

@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+
+class: ExpressionTool
+requirements:
+  - class: InlineJavascriptRequirement
+cwlVersion: v1.2
+
+inputs:
+  file1:
+    type: File
+    loadContents: true
+
+outputs:
+  output: int
+
+expression: "$({'output': parseInt(inputs.file1.contents)})"

testdata/workflows/scatter-wf4.cwl

@@ -0,0 +1,46 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+$graph:
+- id: echo
+  class: CommandLineTool
+  inputs:
+    echo_in1:
+      type: string
+      inputBinding: {}
+    echo_in2:
+      type: string
+      inputBinding: {}
+  outputs:
+    echo_out:
+      type: string
+      outputBinding:
+        glob: "step1_out"
+        loadContents: true
+        outputEval: $(self[0].contents)
+  baseCommand: "echo"
+  arguments: ["-n", "foo"]
+  stdout: step1_out
+
+- id: main
+  class: Workflow
+  inputs:
+    inp1: string[]
+    inp2: string[]
+  requirements:
+    - class: ScatterFeatureRequirement
+  steps:
+    step1:
+      scatter: [echo_in1, echo_in2]
+      scatterMethod: dotproduct
+      in:
+        echo_in1: inp1
+        echo_in2: inp2
+      out: [echo_out]
+      run: "#echo"
+
+  outputs:
+    - id: out
+      outputSource: step1/echo_out
+      type:
+        type: array
+        items: string

testdata/workflows/type-import.yaml

@@ -0,0 +1,2 @@
+- File
+- Directory

testdata/workflows/wc-tool.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.2
+
+inputs:
+  file1: File
+
+outputs:
+  output:
+    type: File
+    outputBinding: { glob: output }
+
+baseCommand: [wc, -l]
+
+stdin: $(inputs.file1.path)
+stdout: output