Merge pull request #114 from crim-ca/fixes-exec-app

fixes some app execution errors

.dockerignore

@@ -64,6 +64,7 @@ reports
 docs
 
 # External Sources
+[Bb]uild
 src
 
 # IPython

.github/workflows/secret-scan.yml

@@ -15,12 +15,12 @@ on:
   - push
 
 jobs:
-  trufflehog:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@master
-    - name: trufflehog-actions-scan
-      uses: edplato/trufflehog-actions-scan@master
+  #trufflehog:
+  #  runs-on: ubuntu-latest
+  #  steps:
+  #  - uses: actions/checkout@master
+  #  - name: trufflehog-actions-scan
+  #    uses: edplato/trufflehog-actions-scan@master
     #- uses: max/secret-scan@master
     #  with:
     #    repo-token: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -54,6 +54,7 @@ testdata.json
 *.sublime*
 
 # External Sources
+[Bb]uild
 src
 
 # IPython

CHANGES.rst

@@ -4,6 +4,17 @@ Changes
 `Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 ========================================================================
 
+Fixes:
+------
+
+- Fix invalid ``AllowedValue`` parsing when using ``LiteralData`` inputs that resulted in ``AnyValue`` being parsed
+  as a ``"None"`` string. This was transparent in case of string inputs and breaking for other types like integer when
+  they attempted conversion.
+- Fix erroneous ``Metadata`` keywords passed down to ``owslib.wps.Metadata`` objects in case of more verbose detailed
+  not allowed by this implementation.
+- Fix parsing of explicitly-typed optional array CWL I/O notation that was not considered
+  (i.e.: using ``type`` as list with additional ``"null"`` instead of ``type: "<type>?"`` shorthand).
+
 `1.5.1 <https://github.com/crim-ca/weaver/tree/1.5.1>`_ (2020-03-26)
 ========================================================================
 

Makefile

@@ -124,7 +124,7 @@ conda-base:		## obtain and install a missing conda distribution
 		(echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)")
 	@test -f "$(CONDA_HOME)/bin/conda" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \
 		(echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \
-		 curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)")
+		 curl "$(CONDA_URL)/$(FN)" --insecure --location --output "$(DOWNLOAD_CACHE)/$(FN)")
 	@test -f "$(CONDA_HOME)/bin/conda" || \
 		(bash "$(DOWNLOAD_CACHE)/$(FN)" -b -u -p "$(CONDA_HOME)" && \
 		 echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.")
@@ -238,6 +238,7 @@ clean-docs-dirs:	## remove documentation artefacts (minimal)
 clean-src:		## remove all *.pyc files
 	@echo "Removing python artifacts..."
 	@-find "$(APP_ROOT)" -type f -name "*.pyc" -exec rm {} \;
+	@-rm -rf ./build
 	@-rm -rf ./src
 
 .PHONY: clean-test

requirements.txt

@@ -1,5 +1,6 @@
 alembic
 argcomplete
+backports.tempfile; python_version < "3"
 celery
 cffi
 colander

tests/functional/test_wps_app.py

@@ -7,8 +7,8 @@
 * http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/testing.html
 """
 import unittest
-from xml.etree import ElementTree
 
+from lxml import etree
 import pyramid.testing
 import pytest
 
@@ -23,6 +23,7 @@
 from weaver.formats import CONTENT_TYPE_ANY_XML
 from weaver.processes.wps_default import HelloWPS
 from weaver.processes.wps_testing import WpsTestProcess
+from weaver.utils import str2bytes
 from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC
 
 
@@ -73,8 +74,8 @@ def test_getcaps_filtered_processes_by_visibility(self):
         assert resp.status_code == 200
         assert resp.content_type in CONTENT_TYPE_ANY_XML
         resp.mustcontain("<wps:ProcessOfferings>")
-        root = ElementTree.fromstring(resp.text)
-        process_offerings = list(filter(lambda e: "ProcessOfferings" in e.tag, list(root)))
+        root = etree.fromstring(str2bytes(resp.text))  # test response has no 'content'
+        process_offerings = list(filter(lambda e: "ProcessOfferings" in e.tag, root.iter(etree.Element)))
         assert len(process_offerings) == 1
         processes = [p for p in process_offerings[0]]
         ids = [pi.text for pi in [list(filter(lambda e: e.tag.endswith("Identifier"), p))[0] for p in processes]]

tests/processes/test_wps_package.py

@@ -213,6 +213,92 @@ def test_is_cwl_array_type_shorthand_enum():
     assert res[3] == ["a", "b", "c"]
 
 
+def test_is_cwl_array_type_explicit_optional_not_array():
+    io_info = {
+        "name": "test",
+        "type": ["null", "float"],
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is False
+    assert res[1] == "float"
+    assert res[2] == MODE.NONE
+    assert res[3] == AnyValue
+
+
+def test_is_cwl_array_type_explicit_optional_simple_enum():
+    io_info = {
+        "name": "test",
+        "type": ["null", "enum"],
+        "symbols": ["a", "b", "c"]
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is False
+    assert res[1] == "enum"
+    assert res[2] == MODE.NONE
+    assert res[3] == AnyValue
+
+
+def test_is_cwl_array_type_explicit_optional_explicit_base():
+    io_info = {
+        "name": "test",
+        "type": [
+            "null",
+            {"type": "array", "items": "string"}
+        ]
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is True
+    assert res[1] == "string"
+    assert res[2] == MODE.NONE
+    assert res[3] == AnyValue
+
+
+def test_is_cwl_array_type_explicit_optional_explicit_enum():
+    io_info = {
+        "name": "test",
+        "type": [
+            "null",
+            {
+                "type": "array",
+                "items": {
+                    "type": "enum",
+                    "symbols": ["a", "b", "c"]
+                }
+            }
+        ]
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is True
+    assert res[1] == "string"
+    assert res[2] == MODE.SIMPLE
+    assert res[3] == ["a", "b", "c"]
+
+
+def test_is_cwl_array_type_explicit_optional_shorthand_base():
+    io_info = {
+        "name": "test",
+        "type": ["null", "string[]"]
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is True
+    assert res[1] == "string"
+    assert res[2] == MODE.NONE
+    assert res[3] == AnyValue
+
+
+def test_is_cwl_array_type_explicit_optional_shorthand_enum():
+    io_info = {
+        "name": "test",
+        "type": ["null", "enum[]"],
+        "symbols": ["a", "b", "c"]
+    }
+    res = _is_cwl_array_type(io_info)
+    assert res[0] is True
+    assert res[1] == "string"
+    assert res[2] == MODE.SIMPLE
+    assert res[3] == ["a", "b", "c"]
+
+
 def test_is_cwl_enum_type_string():
     io_info = {
         "name": "test",

tests/test_utils.py

@@ -21,7 +21,7 @@
 from tests.compat import contextlib
 from tests.utils import mocked_file_response
 from weaver import status, utils
-from weaver.utils import _NullType, null, fetch_file  # noqa: W0212
+from weaver.utils import _NullType, null, fetch_file, make_dirs  # noqa: W0212
 
 
 def test_null_operators():
@@ -76,13 +76,13 @@ def test_get_base_url():
         utils.get_base_url("ftp://localhost:8094/wps")
 
 
-def test_path_elements():
-    assert utils.path_elements("/ows/proxy/lovely_bird") == ["ows", "proxy", "lovely_bird"]
-    assert utils.path_elements("/ows/proxy/lovely_bird/") == ["ows", "proxy", "lovely_bird"]
-    assert utils.path_elements("/ows/proxy/lovely_bird/ ") == ["ows", "proxy", "lovely_bird"]
+def test_xml_path_elements():
+    assert utils.xml_path_elements("/ows/proxy/lovely_bird") == ["ows", "proxy", "lovely_bird"]
+    assert utils.xml_path_elements("/ows/proxy/lovely_bird/") == ["ows", "proxy", "lovely_bird"]
+    assert utils.xml_path_elements("/ows/proxy/lovely_bird/ ") == ["ows", "proxy", "lovely_bird"]
 
 
-def test_lxml_strip_ns():
+def test_xml_strip_ns():
     wps_xml = """
 <wps100:Execute
 xmlns:wps100="http://www.opengis.net/wps/1.0.0"
@@ -93,7 +93,7 @@ def test_lxml_strip_ns():
 
     doc = etree.fromstring(wps_xml)
     assert doc.tag == "{http://www.opengis.net/wps/1.0.0}Execute"
-    utils.lxml_strip_ns(doc)
+    utils.xml_strip_ns(doc)
     assert doc.tag == "Execute"
 
 
@@ -344,7 +344,7 @@ def test_fetch_file_local_with_protocol():
         res_dir = os.path.join(tmp_dir, inspect.currentframe().f_code.co_name)
         res_path = os.path.join(res_dir, tmp_name)
         try:
-            os.makedirs(res_dir, exist_ok=True)
+            make_dirs(res_dir, exist_ok=True)
             for protocol in ["", "file://"]:
                 tmp_path = protocol + tmp_json.name
                 fetch_file(tmp_path, res_dir)
@@ -371,11 +371,13 @@ def test_fetch_file_remote_with_request():
         tmp_http = "http://weaver.mock" + tmp_json.name
         tmp_retry = 2
 
+        # share in below mocked_request, 'nonlocal' back compatible with Python 2
+        tmp = {"retry": tmp_retry, "json": tmp_json, "http": tmp_http}
+
         def mocked_request(*args, **kwargs):  # noqa: E811
-            nonlocal tmp_json, tmp_http, tmp_retry
-            tmp_retry -= 1
-            if not tmp_retry:
-                return mocked_file_response(tmp_json.name, tmp_http)
+            tmp["retry"] -= 1
+            if not tmp["retry"]:
+                return mocked_file_response(tmp["json"].name, tmp["http"])
             resp = Response()
             resp.status_code = HTTPRequestTimeout.code
             return resp  # will be available on next call (to test retries)
@@ -386,7 +388,7 @@ def mocked_request(*args, **kwargs):  # noqa: E811
         res_dir = os.path.join(tmp_dir, inspect.currentframe().f_code.co_name)
         res_path = os.path.join(res_dir, tmp_name)
         try:
-            os.makedirs(res_dir, exist_ok=True)
+            make_dirs(res_dir, exist_ok=True)
             fetch_file(tmp_http, res_dir, retry=tmp_retry + 1)
             assert os.path.isfile(res_path), "File [{}] should be accessible under [{}]".format(tmp_http, res_path)
             assert m_request.call_count == 2, "Request method should have been called twice because of retries"

tests/utils.py

@@ -12,7 +12,7 @@
 import six
 from pyramid import testing
 from pyramid.config import Configurator
-from pyramid.httpexceptions import HTTPNotFound, HTTPUnprocessableEntity, HTTPException
+from pyramid.httpexceptions import HTTPException, HTTPNotFound, HTTPUnprocessableEntity
 from pyramid.registry import Registry
 from requests import Response
 from six.moves.configparser import ConfigParser
@@ -129,7 +129,6 @@ def setup_mongodb_jobstore(config=None):
     config = setup_config_with_mongodb(config)
     store = get_db(config).get_store(MongodbJobStore)
     store.clear_jobs()
-    # noinspection PyTypeChecker
     return store
 
 
@@ -191,7 +190,7 @@ def get_settings_from_testapp(testapp):
     # type: (TestApp) -> SettingsType
     settings = {}
     if hasattr(testapp.app, "registry"):
-        settings = testapp.app.registry.settings or {}
+        settings = testapp.app.registry.settings or {}  # noqa
     return settings
 
 

tests/wps_restapi/test_processes.py

@@ -1,8 +1,8 @@
 import os
 import unittest
 from copy import deepcopy
-import mock
 
+import mock
 import pyramid.testing
 import pytest
 import responses

tests/wps_restapi/test_status_codes.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from tests.utils import setup_config_with_mongodb, get_test_weaver_app
+from tests.utils import get_test_weaver_app, setup_config_with_mongodb
 from weaver.formats import CONTENT_TYPE_APP_JSON
 from weaver.wps_restapi.swagger_definitions import (
     api_frontpage_uri,

weaver/datatype.py

@@ -4,7 +4,7 @@
 import traceback
 import uuid
 from datetime import datetime, timedelta
-from logging import ERROR, INFO, getLevelName, getLogger, Logger
+from logging import ERROR, INFO, Logger, getLevelName, getLogger
 from typing import TYPE_CHECKING
 
 import six

weaver/owsexceptions.py

@@ -106,8 +106,7 @@ class JsonPageTemplate(object):
                     def __init__(self, excobj):
                         self.excobj = excobj
 
-                    # noinspection PyUnusedLocal
-                    def substitute(self, code, locator, message):
+                    def substitute(self, code, locator, message):  # noqa: W0613
                         return json.dumps(self.excobj.json_formatter(
                             status=self.excobj.status, body=message, title=None, environ=environ))
 

weaver/processes/builtin/jsonarray2netcdf.py

@@ -7,12 +7,16 @@
 import logging
 import os
 import sys
-import tempfile
 from typing import Any, AnyStr
 
 import six
 from six.moves.urllib.parse import urlparse
 
+if six.PY3:
+    from tempfile import TemporaryDirectory
+else:
+    from backports.tempfile import TemporaryDirectory
+
 CUR_DIR = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, CUR_DIR)
 # root to allow 'from weaver import <...>'
@@ -52,7 +56,7 @@ def j2n(json_reference, output_dir):
     try:
         if not os.path.isdir(output_dir):
             raise ValueError("Output dir [{}] does not exist.".format(output_dir))
-        with tempfile.TemporaryDirectory(prefix="wps_process_{}_".format(PACKAGE_NAME)) as tmp_dir:
+        with TemporaryDirectory(prefix="wps_process_{}_".format(PACKAGE_NAME)) as tmp_dir:
             LOGGER.debug("Fetching JSON file: [%s]", json_reference)
             json_path = fetch_file(json_reference, tmp_dir, timeout=10, retry=3)
             LOGGER.debug("Reading JSON file: [%s]", json_path)

weaver/processes/opensearch.py

@@ -394,9 +394,8 @@ def make_aoi(id_):
         }
         return data
 
-    # noinspection PyUnusedLocal
     @staticmethod
-    def make_collection(identifier, allowed_values):
+    def make_collection(identifier, allowed_values):  # noqa: W0613
         description = u"Collection of the data."
         data = {
             u"id": u"{}".format(identifier),

weaver/processes/utils.py

@@ -402,10 +402,9 @@ def register_wps_processes_from_config(wps_processes_file_path, container):
                     "executionUnit": [{"href": proc_url}],
                     "deploymentProfileName": "http://www.opengis.net/profiles/eoc/wpsApplication",
                 }
-                # noinspection PyBroadException
                 try:
                     resp = deploy_process_from_payload(payload, container)
-                    if resp.status_code == HTTPOk.status_code:
+                    if resp.status_code == HTTPOk.code:
                         LOGGER.info("Process registered: [%s]", proc_id)
                     else:
                         raise RuntimeError("Process registration failed: [{}]".format(proc_id))