diff --git a/MANIFEST.in b/MANIFEST.in index ad1647692..a53dac497 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -57,6 +57,7 @@ include cwltool/cwlNodeEngineJSConsole.js include cwltool/cwlNodeEngineWithContext.js include cwltool/extensions.yml include cwltool/extensions-v1.1.yml +include cwltool/extensions-v1.2.yml include cwltool/jshint/jshint_wrapper.js include cwltool/jshint/jshint.js include cwltool/hello.simg diff --git a/cwltool/extensions-v1.2.yml b/cwltool/extensions-v1.2.yml new file mode 100644 index 000000000..76c7216ef --- /dev/null +++ b/cwltool/extensions-v1.2.yml @@ -0,0 +1,277 @@ +$base: http://commonwl.org/cwltool# +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + cwltool: "http://commonwl.org/cwltool#" +$graph: +- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml + +- name: Secrets + type: record + inVocab: false + extends: cwl:ProcessRequirement + fields: + class: + type: string + doc: "Always 'Secrets'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + secrets: + type: string[] + doc: | + List one or more input parameters that are sensitive (such as passwords) + which will be deliberately obscured from logging. + jsonldPredicate: + "_type": "@id" + refScope: 0 + + +- name: ProcessGenerator + type: record + inVocab: true + extends: cwl:Process + documentRoot: true + fields: + - name: class + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: string + - name: run + type: [string, cwl:Process] + jsonldPredicate: + _id: "cwl:run" + _type: "@id" + subscope: run + doc: | + Specifies the process to run. + +- name: MPIRequirement + type: record + inVocab: false + extends: cwl:ProcessRequirement + doc: | + Indicates that a process requires an MPI runtime. + fields: + - name: class + type: string + doc: "Always 'MPIRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: processes + type: [int, cwl:Expression] + doc: | + The number of MPI processes to start. If you give a string, + this will be evaluated as a CWL Expression and it must + evaluate to an integer. + +- name: CUDARequirement + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Require support for NVIDA CUDA (GPU hardware acceleration). + fields: + class: + type: string + doc: 'cwltool:CUDARequirement' + jsonldPredicate: + _id: "@type" + _type: "@vocab" + cudaVersionMin: + type: string + doc: | + Minimum CUDA version to run the software, in X.Y format. This + corresponds to a CUDA SDK release. When running directly on + the host (not in a container) the host must have a compatible + CUDA SDK (matching the exact version, or, starting with CUDA + 11.3, matching major version). When run in a container, the + container image should provide the CUDA runtime, and the host + driver is injected into the container. In this case, because + CUDA drivers are backwards compatible, it is possible to + use an older SDK with a newer driver across major versions. + + See https://docs.nvidia.com/deploy/cuda-compatibility/ for + details. + cudaComputeCapability: + type: + - 'string' + - 'string[]' + doc: | + CUDA hardware capability required to run the software, in X.Y + format. + + * If this is a single value, it defines only the minimum + compute capability. GPUs with higher capability are also + accepted. + + * If it is an array value, then only select GPUs with compute + capabilities that explicitly appear in the array. + cudaDeviceCountMin: + type: ['null', int, cwl:Expression] + default: 1 + doc: | + Minimum number of GPU devices to request. If not specified, + same as `cudaDeviceCountMax`. If neither are specified, + default 1. + cudaDeviceCountMax: + type: ['null', int, cwl:Expression] + doc: | + Maximum number of GPU devices to request. If not specified, + same as `cudaDeviceCountMin`. + +- name: intervalBase + type: record + abstract: true + fields: + low: + type: [int, float, double] + default: -.inf # negative infinity + jsonldPredicate: "cwltool:low" + high: + type: [int, float, double] + default: .inf # positive infinity + jsonldPredicate: "cwltool:high" + +- name: intInterval + type: record + extends: intervalBase + doc: | + Integer number interval specification. All integer intervals are inclusive. + fields: + class: + type: + type: enum + name: intInterval_class + symbols: + - cwltool:intInterval + jsonldPredicate: + _id: "@type" + _type: "@vocab" + low: + type: int + default: -.inf # negative infinity + jsonldPredicate: "cwltool:low" + high: + type: int + default: .inf # positive infinity + jsonldPredicate: "cwltool:high" + # compact form proposal + # doc: | + # Examples: + # "[0,3)" any real number between 0 (inclusive) and 3 (exclusive) + # "[6,)" any real number greater than or equal to 6 + # "(0,1)" any real number between 0 and 1 (exclusive) + + +- name: realInterval + type: record + extends: intervalBase + doc: | + Integer number interval + fields: + class: + type: + type: enum + name: realInterval_class + symbols: + - cwltool:realInterval + jsonldPredicate: + _id: "@type" + _type: "@vocab" + low_inclusive: + type: boolean + default: true + high_inclusive: + type: boolean + default: true + # compact form proposal + # doc: | + # Use "(" or "[" to indicate an exclusive or inclusive beginning, + # and ")" or "]" to indicate an exclusive or inclusive end. + # Default is inclusive. + # Examples: + # "[0,3)" any real number between 0 (inclusive) and 3 (exclusive) + # "[6,)" any real number greater than or equal to 6 + # "(0,1)" any real number between 0 and 1 (exclusive) + +- name: regex + type: record + doc: | + ECMAScript 5.1 Regular Expression constraint. + fields: + class: + type: + type: enum + name: regex_class + symbols: + - cwltool:regex + jsonldPredicate: + _id: "@type" + _type: "@vocab" + rpattern: + type: string + doc: | + Testing should be the equivalent of calling `/rpattern/.test(value)` + where `rpattern` is the value of the `rpattern` field, and `value` + is the input object value to be tested. If the result is `true` then the + input object value is accepted. If the result if `false` then the + input object value does not match this constraint. + + +- name: Restrictions + type: record + fields: + input: + type: string + jsonldPredicate: + "_type": "@id" + refScope: 2 + constraints: + type: + type: array + items: + - string + - int + - float + - double + - regex + - cwl:Expression + - intInterval + - realInterval + +- name: ParameterRestrictions + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Prototype of input value restrictions construct. + fields: + class: + type: + type: enum + name: ParameterRestrictions_class + symbols: + - cwltool:ParameterRestrictions + jsonldPredicate: + _id: "@type" + _type: "@vocab" + restrictions: + type: Restrictions[] + jsonldPredicate: + _id: "cwltool:restrictions" + mapSubject: input + mapPredicate: constraints + doc: | + (Only applicable for string, int, long, float and double parameters) + List of restrictions that apply to the input parameter "id". + A given parameter value should be accepted if any of the restrictions + match the input. Restrictions can be + single values (applies to all types), + ranges of the same type (applies to int, float and double), + regular expressions (for string parameters) + or Expressions (if an expression is provided, the expression must + return a boolean representing a match). + If the parameter is an array type, every value has to match at least + one of the restrictions. diff --git a/cwltool/main.py b/cwltool/main.py index df7b0f748..0adec86c8 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -661,9 +661,11 @@ def setup_schema( ext10 = res.read().decode("utf-8") with pkg_resources.resource_stream(__name__, "extensions-v1.1.yml") as res: ext11 = res.read().decode("utf-8") + with pkg_resources.resource_stream(__name__, "extensions-v1.2.yml") as res: + ext12 = res.read().decode("utf-8") use_custom_schema("v1.0", "http://commonwl.org/cwltool", ext10) use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11) - use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext11) + use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext12) use_custom_schema("v1.2.0-dev1", "http://commonwl.org/cwltool", ext11) use_custom_schema("v1.2.0-dev2", "http://commonwl.org/cwltool", ext11) use_custom_schema("v1.2.0-dev3", "http://commonwl.org/cwltool", ext11) diff --git a/cwltool/process.py b/cwltool/process.py index 9abbfd704..62dce1d51 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -20,6 +20,7 @@ Iterable, Iterator, List, + Mapping, MutableMapping, MutableSequence, Optional, @@ -55,6 +56,7 @@ from .loghandler import _logger from .mpi import MPIRequirementName from .pathmapper import MapperEnt, PathMapper +from .sandboxjs import execjs from .secrets import SecretStore from .stdfsaccess import StdFsAccess from .update import INTERNAL_VERSION, ORIGINAL_CWLVERSION @@ -121,6 +123,7 @@ def filter(self, record: logging.LogRecord) -> bool: "http://commonwl.org/cwltool#LoadListingRequirement", "http://commonwl.org/cwltool#InplaceUpdateRequirement", "http://commonwl.org/cwltool#CUDARequirement", + "http://commonwl.org/cwltool#ParameterRestrictions", ] cwl_files = ( @@ -522,10 +525,11 @@ def var_spool_cwl_detector( r = False if isinstance(obj, str): if "var/spool/cwl" in obj and obj_key != "dockerOutputDirectory": + debug = _logger.isEnabledFor(logging.DEBUG) _logger.warning( - SourceLine(item=item, key=obj_key, raise_type=str).makeError( - _VAR_SPOOL_ERROR.format(obj) - ) + SourceLine( + item=item, key=obj_key, raise_type=str, include_traceback=debug + ).makeError(_VAR_SPOOL_ERROR.format(obj)) ) r = True elif isinstance(obj, MutableMapping): @@ -742,7 +746,9 @@ def __init__( and not is_req ): _logger.warning( - SourceLine(item=dockerReq, raise_type=str).makeError( + SourceLine( + item=dockerReq, raise_type=str, include_traceback=debug + ).makeError( "When 'dockerOutputDirectory' is declared, DockerRequirement " "should go in the 'requirements' section, not 'hints'." "" @@ -804,6 +810,98 @@ def _init_job( vocab=INPUT_OBJ_VOCAB, ) + restriction_req, mandatory_restrictions = self.get_requirement( + "ParameterRestrictions" + ) + + if restriction_req: + restrictions = restriction_req["restrictions"] + for entry in cast(List[CWLObjectType], restrictions): + name = shortname(cast(str, entry["input"])) + if name in job: + value = job[name] + matched = False + for constraint in cast( + List[CWLOutputType], entry["constraints"] + ): + if isinstance(constraint, Mapping): + if constraint["class"] == "intInterval": + if not isinstance(value, int): + raise SourceLine( + constraint, + None, + WorkflowException, + runtime_context.debug, + ).makeError( + "intInterval parameter restriction is only valid for inputs of type 'int'; " + f"instead got {type(value)}: {value}." + ) + low = cast( + Union[int, float], + constraint.get("low", -math.inf), + ) + high = cast( + Union[int, float], + constraint.get("high", math.inf), + ) + matched = value >= low and value <= high + elif constraint["class"] == "realInterval": + if not isinstance(value, (int, float)): + raise SourceLine( + constraint, + None, + WorkflowException, + runtime_context.debug, + ).makeError( + "realInterval parameter restriction is only valid for inputs of type 'int', 'float', and 'double'; " + f"instead got {type(value)}: {value}." + ) + low = cast( + Union[int, float], + constraint.get("low", -math.inf), + ) + high = cast( + Union[int, float], + constraint.get("high", math.inf), + ) + low_inclusive = constraint.get( + "low_inclusive", True + ) + high_inclusive = constraint.get( + "high_inclusive", True + ) + check_low = ( + value >= low if low_inclusive else value > low + ) + check_high = ( + value <= high if low_inclusive else value < high + ) + matched = check_low and check_high + elif constraint["class"] == "regex": + rpattern = constraint["rpattern"] + quoted_value = json.dumps(value) + matched = cast( + bool, + execjs( + f"/{rpattern}/.test({quoted_value})", + "", + runtime_context.eval_timeout, + runtime_context.force_docker_pull, + ), + ) + elif constraint == value: + matched = True + if matched: + break + if not matched: + raise SourceLine( + job, name, WorkflowException, runtime_context.debug + ).makeError( + f"The field '{name}' is not valid because its " + f"value '{value}' failed to match any of the " + f"constraints '{json.dumps(entry['constraints'])}'." + ) + if load_listing and load_listing != "no_listing": get_listing(fs_access, job, recursive=(load_listing == "deep_listing")) diff --git a/requirements.txt b/requirements.txt index 34c8ec114..548381f7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests>=2.4.3 ruamel.yaml>=0.15,<0.17.22 rdflib>=4.2.2,<6.2 shellescape>=3.4.1,<3.9 -schema-salad>=8.2.20211104054942,<9 +schema-salad>=8.3,<9 prov==1.5.1 bagit==1.8.1 mypy-extensions diff --git a/setup.py b/setup.py index c3bb9a381..c99f53e33 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ "ruamel.yaml >= 0.15, < 0.17.22", "rdflib >= 4.2.2, < 6.2.0", "shellescape >= 3.4.1, < 3.9", - "schema-salad >= 8.2.20211104054942, < 9", + "schema-salad >= 8.3, < 9", "mypy-extensions", "psutil >= 5.6.6", "prov == 1.5.1", diff --git a/tests/parameter_restrictions.cwl b/tests/parameter_restrictions.cwl new file mode 100644 index 000000000..2e5a5f6e9 --- /dev/null +++ b/tests/parameter_restrictions.cwl @@ -0,0 +1,45 @@ +cwlVersion: v1.2 +class: CommandLineTool + +doc: | + Tests of paramater restrictions extension + +inputs: + one: double + two: string + three: int + + +hints: + cwltool:ParameterRestrictions: + restrictions: + one: + - class: cwltool:realInterval + low: 0 + high: 3 + high_inclusive: false + - class: cwltool:realInterval + low: 6 + # high should be the default of positive infinity + high_inclusive: false + two: + - class: cwltool:regex + rpattern: "foo.*bar" + three: + - class: cwltool:intInterval + low: -10 + high: -7 + + +baseCommand: echo + +arguments: + - $(inputs.one) + - $(inputs.two) + +outputs: + result: stdout + +$namespaces: + cwltool: "http://commonwl.org/cwltool#" + diff --git a/tests/test_ext.py b/tests/test_ext.py index 1eb4091e5..6a9bd6743 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -295,3 +295,51 @@ def test_warn_large_inputs() -> None: ) finally: cwltool.process.FILE_COUNT_WARNING = was + + +def test_parameter_restrictions_parsing() -> None: + """Basic parsing of the ParameterRestrictions extension.""" + assert ( + main( + ["--enable-ext", "--validate", get_data("tests/parameter_restrictions.cwl")] + ) + == 0 + ) + + +def test_parameter_restrictions_valid_job() -> None: + """Confirm that valid values are accepted.""" + assert ( + main( + [ + "--enable-ext", + get_data("tests/parameter_restrictions.cwl"), + "--one", + "2.5", + "--two", + "foofoobar", + "--three", + "-5", + ] + ) + == 0 + ) + + +def test_parameter_restrictions_invalid_job() -> None: + """Confirm that an invvalid value is rejected.""" + assert ( + main( + [ + "--enable-ext", + get_data("tests/parameter_restrictions.cwl"), + "--one", + "2.5", + "--two", + "fuzzbar", + "--three", + "-5", + ] + ) + == 1 + )