Update tests running scripts

  - Use versioned docker images
  - Exclude tests from coverage run

.github/workflows/config.yml

@@ -197,16 +197,6 @@ jobs:
       - name: Build the Docker Image
         run: scripts/prepare_image.sh -i base -s common
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_common:
     name: Test Common
     env:
@@ -248,16 +238,6 @@ jobs:
           common/run_tests.sh
           .github/scripts/upload_test_coverage.sh common
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_downloaders_no_op:
     name: Test Downloaders and No-Op
     env:
@@ -336,16 +316,6 @@ jobs:
           foreman/run_tests.sh --exclude-tag=end_to_end
           .github/scripts/upload_test_coverage.sh foreman
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_illumina:
     name: Test Illumina
     env:
@@ -381,16 +351,6 @@ jobs:
           .github/scripts/filter_tests.sh -t illumina
           .github/scripts/upload_test_coverage.sh workers
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_salmon:
     name: Test Salmon
     env:
@@ -425,16 +385,6 @@ jobs:
           .github/scripts/filter_tests.sh -t salmon
           .github/scripts/upload_test_coverage.sh workers
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_smasher:
     name: Test Smasher
     env:
@@ -488,16 +438,6 @@ jobs:
           .github/scripts/filter_tests.sh -t qn
           .github/scripts/upload_test_coverage.sh workers
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   test_transcriptome:
     name: Test Transcriptome
     env:
@@ -534,16 +474,6 @@ jobs:
           .github/scripts/filter_tests.sh -t transcriptome
           .github/scripts/upload_test_coverage.sh workers
 
-      - name: Login to Packages Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Push Built Docker Images
-        run: .github/scripts/push_docker_images.sh
-
   # Deploy jobs.
   deploy:
     name: Deploy

api/dockerfiles/Dockerfile.api_base

@@ -23,6 +23,7 @@ RUN pip install --ignore-installed \
 
 COPY api/ .
 COPY config/ config/
+COPY setup.cfg .
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

api/run_tests.sh

@@ -35,6 +35,7 @@ if [ ! -d "$volume_directory" ]; then
 fi
 chmod -R a+rwX "$volume_directory"
 
+./scripts/prepare_image.sh -i api_base -s api
 ./scripts/prepare_image.sh -i api_local -s api
 
 . ./scripts/common.sh
@@ -56,5 +57,5 @@ docker run \
     --tty \
     --volume "$volume_directory":/home/user/data_store \
     $INTERACTIVE \
-    "$DOCKERHUB_REPO/dr_api_local" \
+    "$DOCKERHUB_REPO/dr_api_local:$SYSTEM_VERSION" \
     bash -c "$(run_tests_with_coverage "$@")"

common/dockerfiles/Dockerfile.base

@@ -66,4 +66,7 @@ RUN apt-get update && \
 
 ENV R_LIBS=/usr/local/lib/R/site-library
 
+COPY config/ config/
+COPY setup.cfg .
+
 ENTRYPOINT []

common/dockerfiles/Dockerfile.common_tests

@@ -15,7 +15,6 @@ COPY common/requirements.txt .
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
 COPY common/ .
-COPY config config
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

common/dockerfiles/Dockerfile.migrations

@@ -11,7 +11,6 @@ COPY common/requirements.txt .
 RUN pip install --ignore-installed --no-cache-dir -r requirements.txt
 
 COPY common/ .
-COPY config config
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

common/run_tests.sh

@@ -28,6 +28,7 @@ if [ ! -d "$volume_directory" ]; then
 fi
 chmod -R a+rwX "$volume_directory"
 
+./scripts/prepare_image.sh -i base -s common
 ./scripts/prepare_image.sh -i common_tests -s common
 
 . ./scripts/common.sh
@@ -49,5 +50,5 @@ docker run \
     --tty \
     --volume "$volume_directory":/home/user/data_store \
     $INTERACTIVE \
-    "$DOCKERHUB_REPO/dr_common_tests" \
+    "$DOCKERHUB_REPO/dr_common_tests:$SYSTEM_VERSION" \
     bash -c "$(run_tests_with_coverage "$@")" --parallel

foreman/dockerfiles/Dockerfile.foreman

@@ -21,7 +21,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config config
 COPY foreman/ .
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION

foreman/run_tests.sh

@@ -32,6 +32,7 @@ if ! [ "$(docker ps --filter name=drdb -q)" ]; then
     exit 1
 fi
 
+./scripts/prepare_image.sh -i base -s common
 ./scripts/prepare_image.sh -i foreman -s foreman
 
 . ./scripts/common.sh
@@ -51,5 +52,5 @@ docker run \
     --tty \
     --volume "$volume_directory":/home/user/data_store \
     $INTERACTIVE \
-    "$DOCKERHUB_REPO/dr_foreman" \
+    "$DOCKERHUB_REPO/dr_foreman:$SYSTEM_VERSION" \
     bash -c "$(run_tests_with_coverage --exclude-tag=manual "$@")"

scripts/common.sh

@@ -6,11 +6,6 @@ export ALL_IMAGES="smasher compendia illumina affymetrix salmon transcriptome no
 # Sometimes we only want to work with the worker images.
 export WORKER_IMAGES="smasher compendia illumina affymetrix salmon transcriptome no_op downloaders"
 
-# Default Docker registry.
-if [ -z "$DOCKERHUB_REPO" ]; then
-    export DOCKERHUB_REPO="ccdlstaging"
-fi
-
 # This function checks whether a given docker image name ($1:$2)
 # exists in Docker Hub or not using Docker Hub API V2. Based on:
 # https://stackoverflow.com/questions/32113330/check-if-imagetag-combination-already-exists-on-docker-hub
@@ -118,3 +113,13 @@ EOF
     echo "Using Docker builder $DOCKER_BUILDER."
     docker buildx use "$DOCKER_BUILDER"
 }
+
+# Default Docker registry.
+if [ -z "$DOCKERHUB_REPO" ]; then
+    export DOCKERHUB_REPO="ccdlstaging"
+fi
+
+# Defaults to commit hash value for if we're not running in the cloud.
+if [ -z "$SYSTEM_VERSION" ]; then
+    SYSTEM_VERSION="$(get_branch_hash)"
+fi

scripts/prepare_image.sh

@@ -86,7 +86,7 @@ if [ -z "$SYSTEM_VERSION" ]; then
 fi
 
 if [ -z "$DOCKER_ACTION" ]; then
-    DOCKER_ACTION="--load"
+    DOCKER_ACTION="--push"
 fi
 
 DOCKERHUB_IMAGE="$DOCKERHUB_REPO/dr_$IMAGE_NAME"
@@ -141,8 +141,6 @@ if [ $finished -ne 0 ] && [ $attempt -ge $attempts ]; then
     exit 1
 fi
 
-if test "$GITHUB_ACTION"; then
-    docker pull \
-        --platform linux/amd64 \
-        "$DOCKERHUB_IMAGE"
-fi
+docker pull \
+    --platform linux/amd64 \
+    "$DOCKERHUB_IMAGE:$SYSTEM_VERSION"

setup.cfg

@@ -1,8 +1,11 @@
-[coverage:report]
-include =
-    data_refinery_api/*
+[coverage:run]
 omit =
-    data_refinery_api/wsgi.py
+    tests/*
+    manage.py
+    settings.py
+    setup.py
+    urls.py
+    wsgi.py
 
 [pep8]
 max-line-length = 100

workers/dockerfiles/Dockerfile.affymetrix

@@ -23,7 +23,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.compendia

@@ -92,6 +92,7 @@ RUN pip3 install --ignore-installed --no-cache-dir \
 
 COPY .boto .boto
 COPY config/ config/
+COPY setup.cfg .
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.downloaders

@@ -31,7 +31,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config config
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.illumina

@@ -20,7 +20,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 COPY workers/data_refinery_workers/processors/detect_database.R .
 COPY workers/illumina_probe_maps/ probe_maps/

workers/dockerfiles/Dockerfile.no_op

@@ -32,7 +32,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.salmon

@@ -56,7 +56,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.smasher

@@ -20,7 +20,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/dockerfiles/Dockerfile.transcriptome

@@ -38,7 +38,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \
         common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*

workers/run_tests.sh

@@ -529,6 +529,8 @@ fi
 
 . ./scripts/common.sh
 
+./scripts/prepare_image.sh -i base -s common
+
 DB_HOST_IP=$(get_docker_db_ip_address)
 
 # Ensure permissions are set for everything within the test data directory.
@@ -577,7 +579,7 @@ for image in $worker_images; do
             --tty \
             --volume "$volume_directory":/home/user/data_store \
             $INTERACTIVE \
-            "$image_name" \
+            "$image_name:$SYSTEM_VERSION" \
             bash -c "$test_command"
     fi
 done