run CI builds as non-root user (#293)

It's useful to run the test suite as some arbitrary uid so that the runner has realistic file permissions constraints.

.travis.yml

@@ -14,7 +14,7 @@ script:
       set -exo pipefail
       make docker
       docker run --env TRAVIS_JOB_ID=${TRAVIS_JOB_ID} --env TRAVIS_BRANCH=${TRAVIS_BRANCH} \
-                  -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
+                  --group-add $(stat -c %g /var/run/docker.sock) -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
                   miniwdl bash -c "make $CI_TARGET && coveralls"
 
 jobs:

Dockerfile

@@ -1,19 +1,31 @@
-# Start with ubuntu:18.04 plus some apt packages
+# builds docker image for running test suite for the contextual miniwdl source tree
+#    docker build -t miniwdl .
+# run the full test suite -- notice configuration needed for it to command the host dockerd
+#    docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock --group-add $(stat -c %g /var/run/docker.sock) -v /tmp:/tmp miniwdl
+# or append 'bash' to that to enter interactive shell
+
+# start with ubuntu:18.04 plus some apt packages
 FROM ubuntu:18.04
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \
-    python3 python3-pip python3-setuptools tzdata wget zip git-core \
-    default-jre jq graphviz shellcheck docker.io
-# pip install the requirements files for run & dev
-COPY requirements.txt requirements.dev.txt /miniwdl/
-RUN bash -o pipefail -c "pip3 install --user -r <(cat /miniwdl/requirements.txt /miniwdl/requirements.dev.txt)"
-# Copy in the local source tree / build context. We've delayed this until after
-# requirements so that docker build doesn't reinstall the pip packages on every
-# minor source change.
-ADD . /miniwdl
+    python3-pip python3-setuptools tzdata wget zip git-core default-jre jq shellcheck docker.io
+
+# add and become 'wdler' user -- it's useful to run the test suite as some arbitrary uid, because
+# the runner has numerous file permissions-related constraints
+RUN useradd -ms /bin/bash -u 1337 wdler
+USER wdler
+
+# pip install the requirements files -- we do this before adding the rest of the source tree, so
+# that docker build doesn't have to reinstall the pip packages for every minor source change
+COPY requirements.txt requirements.dev.txt /home/wdler/
+RUN bash -o pipefail -c "pip3 install --user -r <(cat /home/wdler/requirements.txt /home/wdler/requirements.dev.txt)"
+
+# add the source tree
+ADD --chown=wdler:wdler . /miniwdl
 WORKDIR /miniwdl
-ENV PYTHONPATH $PYTHONPATH:/root/.local/lib/python3.6
-ENV PATH $PATH:/root/.local/bin
-# will trigger typechecking & tests:
+
+# finishing touches
+ENV PYTHONPATH $PYTHONPATH:/home/wdler/.local/lib/python3.6
+ENV PATH $PATH:/home/wdler/.local/bin
 CMD make

WDL/Tree.py

@@ -1581,7 +1581,7 @@ def _decl_dependency_matrix(decls: List[Decl]) -> Tuple[Dict[str, Decl], _util.A
 
 
 def _workflow_dependency_matrix(
-    workflow: Workflow
+    workflow: Workflow,
 ) -> Tuple[Dict[str, WorkflowNode], _util.AdjM[str]]:
     # Given workflow, produce mapping of workflow node id to each node, and the AdjM of their
     # dependencies (edge from o1 to o2 = o2 depends on o1). Considers each Scatter and Conditional

requirements.dev.txt

@@ -1,7 +1,7 @@
 # Packages needed for miniwdl development, in addition to those in
 # requirements.txt which are needed for miniwdl to run in common use.
 pyre-check==0.0.27
-black==19.3b0
+black==19.10b0
 pylint
 sphinx
 sphinx-autobuild

tests/test_4taskrun.py

@@ -747,6 +747,7 @@ def test_runtime_memory(self):
             >>>
             runtime {
                 memory: "~{memory}"
+                disks: "ignored"
             }
         }
         """