diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 902cd0765..11197af48 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -197,16 +197,6 @@ jobs: - name: Build the Docker Image run: scripts/prepare_image.sh -i base -s common - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_common: name: Test Common env: @@ -248,16 +238,6 @@ jobs: common/run_tests.sh .github/scripts/upload_test_coverage.sh common - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_downloaders_no_op: name: Test Downloaders and No-Op env: @@ -336,16 +316,6 @@ jobs: foreman/run_tests.sh --exclude-tag=end_to_end .github/scripts/upload_test_coverage.sh foreman - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_illumina: name: Test Illumina env: @@ -381,16 +351,6 @@ jobs: .github/scripts/filter_tests.sh -t illumina .github/scripts/upload_test_coverage.sh workers - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_salmon: name: Test Salmon env: @@ -425,16 +385,6 @@ jobs: .github/scripts/filter_tests.sh -t salmon .github/scripts/upload_test_coverage.sh workers - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_smasher: name: Test Smasher env: @@ -488,16 +438,6 @@ jobs: .github/scripts/filter_tests.sh -t qn .github/scripts/upload_test_coverage.sh workers - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - test_transcriptome: name: Test Transcriptome env: @@ -534,16 +474,6 @@ jobs: .github/scripts/filter_tests.sh -t transcriptome .github/scripts/upload_test_coverage.sh workers - - name: Login to Packages Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push Built Docker Images - run: .github/scripts/push_docker_images.sh - # Deploy jobs. deploy: name: Deploy diff --git a/api/dockerfiles/Dockerfile.api_base b/api/dockerfiles/Dockerfile.api_base index d1d751104..44bf4c787 100644 --- a/api/dockerfiles/Dockerfile.api_base +++ b/api/dockerfiles/Dockerfile.api_base @@ -23,6 +23,7 @@ RUN pip install --ignore-installed \ COPY api/ . COPY config/ config/ +COPY setup.cfg . ENV SYSTEM_VERSION=$SYSTEM_VERSION diff --git a/api/run_tests.sh b/api/run_tests.sh index f4353c679..0b4072131 100755 --- a/api/run_tests.sh +++ b/api/run_tests.sh @@ -35,6 +35,7 @@ if [ ! -d "$volume_directory" ]; then fi chmod -R a+rwX "$volume_directory" +./scripts/prepare_image.sh -i api_base -s api ./scripts/prepare_image.sh -i api_local -s api . ./scripts/common.sh @@ -56,5 +57,5 @@ docker run \ --tty \ --volume "$volume_directory":/home/user/data_store \ $INTERACTIVE \ - "$DOCKERHUB_REPO/dr_api_local" \ + "$DOCKERHUB_REPO/dr_api_local:$SYSTEM_VERSION" \ bash -c "$(run_tests_with_coverage "$@")" diff --git a/common/dockerfiles/Dockerfile.base b/common/dockerfiles/Dockerfile.base index 13987e894..ef5d13c3b 100644 --- a/common/dockerfiles/Dockerfile.base +++ b/common/dockerfiles/Dockerfile.base @@ -66,4 +66,7 @@ RUN apt-get update && \ ENV R_LIBS=/usr/local/lib/R/site-library +COPY config/ config/ +COPY setup.cfg . + ENTRYPOINT [] diff --git a/common/dockerfiles/Dockerfile.common_tests b/common/dockerfiles/Dockerfile.common_tests index 26b2be946..d914a52f0 100644 --- a/common/dockerfiles/Dockerfile.common_tests +++ b/common/dockerfiles/Dockerfile.common_tests @@ -15,7 +15,6 @@ COPY common/requirements.txt . RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt COPY common/ . -COPY config config ENV SYSTEM_VERSION=$SYSTEM_VERSION diff --git a/common/dockerfiles/Dockerfile.migrations b/common/dockerfiles/Dockerfile.migrations index da2405265..516ce642b 100644 --- a/common/dockerfiles/Dockerfile.migrations +++ b/common/dockerfiles/Dockerfile.migrations @@ -11,7 +11,6 @@ COPY common/requirements.txt . RUN pip install --ignore-installed --no-cache-dir -r requirements.txt COPY common/ . -COPY config config ENV SYSTEM_VERSION=$SYSTEM_VERSION diff --git a/common/run_tests.sh b/common/run_tests.sh index 41eb8e3d1..25af0edb3 100755 --- a/common/run_tests.sh +++ b/common/run_tests.sh @@ -28,6 +28,7 @@ if [ ! -d "$volume_directory" ]; then fi chmod -R a+rwX "$volume_directory" +./scripts/prepare_image.sh -i base -s common ./scripts/prepare_image.sh -i common_tests -s common . ./scripts/common.sh @@ -49,5 +50,5 @@ docker run \ --tty \ --volume "$volume_directory":/home/user/data_store \ $INTERACTIVE \ - "$DOCKERHUB_REPO/dr_common_tests" \ + "$DOCKERHUB_REPO/dr_common_tests:$SYSTEM_VERSION" \ bash -c "$(run_tests_with_coverage "$@")" --parallel diff --git a/foreman/dockerfiles/Dockerfile.foreman b/foreman/dockerfiles/Dockerfile.foreman index 89e3611f0..026f5159f 100644 --- a/foreman/dockerfiles/Dockerfile.foreman +++ b/foreman/dockerfiles/Dockerfile.foreman @@ -21,7 +21,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config config COPY foreman/ . ENV SYSTEM_VERSION=$SYSTEM_VERSION diff --git a/foreman/run_tests.sh b/foreman/run_tests.sh index f0ae37de4..7ec565510 100755 --- a/foreman/run_tests.sh +++ b/foreman/run_tests.sh @@ -32,6 +32,7 @@ if ! [ "$(docker ps --filter name=drdb -q)" ]; then exit 1 fi +./scripts/prepare_image.sh -i base -s common ./scripts/prepare_image.sh -i foreman -s foreman . ./scripts/common.sh @@ -51,5 +52,5 @@ docker run \ --tty \ --volume "$volume_directory":/home/user/data_store \ $INTERACTIVE \ - "$DOCKERHUB_REPO/dr_foreman" \ + "$DOCKERHUB_REPO/dr_foreman:$SYSTEM_VERSION" \ bash -c "$(run_tests_with_coverage --exclude-tag=manual "$@")" diff --git a/scripts/common.sh b/scripts/common.sh index b30416abc..f995b801a 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -6,11 +6,6 @@ export ALL_IMAGES="smasher compendia illumina affymetrix salmon transcriptome no # Sometimes we only want to work with the worker images. export WORKER_IMAGES="smasher compendia illumina affymetrix salmon transcriptome no_op downloaders" -# Default Docker registry. -if [ -z "$DOCKERHUB_REPO" ]; then - export DOCKERHUB_REPO="ccdlstaging" -fi - # This function checks whether a given docker image name ($1:$2) # exists in Docker Hub or not using Docker Hub API V2. Based on: # https://stackoverflow.com/questions/32113330/check-if-imagetag-combination-already-exists-on-docker-hub @@ -118,3 +113,13 @@ EOF echo "Using Docker builder $DOCKER_BUILDER." docker buildx use "$DOCKER_BUILDER" } + +# Default Docker registry. +if [ -z "$DOCKERHUB_REPO" ]; then + export DOCKERHUB_REPO="ccdlstaging" +fi + +# Defaults to commit hash value for if we're not running in the cloud. +if [ -z "$SYSTEM_VERSION" ]; then + SYSTEM_VERSION="$(get_branch_hash)" +fi diff --git a/scripts/prepare_image.sh b/scripts/prepare_image.sh index d9a49c68d..4aee4b79e 100755 --- a/scripts/prepare_image.sh +++ b/scripts/prepare_image.sh @@ -86,7 +86,7 @@ if [ -z "$SYSTEM_VERSION" ]; then fi if [ -z "$DOCKER_ACTION" ]; then - DOCKER_ACTION="--load" + DOCKER_ACTION="--push" fi DOCKERHUB_IMAGE="$DOCKERHUB_REPO/dr_$IMAGE_NAME" @@ -141,8 +141,6 @@ if [ $finished -ne 0 ] && [ $attempt -ge $attempts ]; then exit 1 fi -if test "$GITHUB_ACTION"; then - docker pull \ - --platform linux/amd64 \ - "$DOCKERHUB_IMAGE" -fi +docker pull \ + --platform linux/amd64 \ + "$DOCKERHUB_IMAGE:$SYSTEM_VERSION" diff --git a/setup.cfg b/setup.cfg index 8b9a8a747..a4b2eb30b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,11 @@ -[coverage:report] -include = - data_refinery_api/* +[coverage:run] omit = - data_refinery_api/wsgi.py + tests/* + manage.py + settings.py + setup.py + urls.py + wsgi.py [pep8] max-line-length = 100 diff --git a/workers/dockerfiles/Dockerfile.affymetrix b/workers/dockerfiles/Dockerfile.affymetrix index 8dad322b3..69983d365 100644 --- a/workers/dockerfiles/Dockerfile.affymetrix +++ b/workers/dockerfiles/Dockerfile.affymetrix @@ -23,7 +23,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.compendia b/workers/dockerfiles/Dockerfile.compendia index 371abf2c7..23957c892 100644 --- a/workers/dockerfiles/Dockerfile.compendia +++ b/workers/dockerfiles/Dockerfile.compendia @@ -92,6 +92,7 @@ RUN pip3 install --ignore-installed --no-cache-dir \ COPY .boto .boto COPY config/ config/ +COPY setup.cfg . COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.downloaders b/workers/dockerfiles/Dockerfile.downloaders index 5361db49f..284203b35 100644 --- a/workers/dockerfiles/Dockerfile.downloaders +++ b/workers/dockerfiles/Dockerfile.downloaders @@ -31,7 +31,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config config COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.illumina b/workers/dockerfiles/Dockerfile.illumina index 1d771b322..ae00c06ca 100644 --- a/workers/dockerfiles/Dockerfile.illumina +++ b/workers/dockerfiles/Dockerfile.illumina @@ -20,7 +20,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . COPY workers/data_refinery_workers/processors/detect_database.R . COPY workers/illumina_probe_maps/ probe_maps/ diff --git a/workers/dockerfiles/Dockerfile.no_op b/workers/dockerfiles/Dockerfile.no_op index 7e33a14e3..f37a21ce4 100644 --- a/workers/dockerfiles/Dockerfile.no_op +++ b/workers/dockerfiles/Dockerfile.no_op @@ -32,7 +32,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.salmon b/workers/dockerfiles/Dockerfile.salmon index 08a3d3299..6fb5885cb 100644 --- a/workers/dockerfiles/Dockerfile.salmon +++ b/workers/dockerfiles/Dockerfile.salmon @@ -56,7 +56,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.smasher b/workers/dockerfiles/Dockerfile.smasher index f95da5016..e04410251 100644 --- a/workers/dockerfiles/Dockerfile.smasher +++ b/workers/dockerfiles/Dockerfile.smasher @@ -20,7 +20,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/dockerfiles/Dockerfile.transcriptome b/workers/dockerfiles/Dockerfile.transcriptome index e3c5328a4..a27cc15aa 100644 --- a/workers/dockerfiles/Dockerfile.transcriptome +++ b/workers/dockerfiles/Dockerfile.transcriptome @@ -38,7 +38,6 @@ RUN pip3 install --ignore-installed --no-cache-dir \ common/$(ls common -1 | sort --version-sort | tail -1) COPY .boto .boto -COPY config/ config/ COPY workers/ . RUN rm -rf /root/.cache/* diff --git a/workers/run_tests.sh b/workers/run_tests.sh index a66d0a2f7..092f189bc 100755 --- a/workers/run_tests.sh +++ b/workers/run_tests.sh @@ -529,6 +529,8 @@ fi . ./scripts/common.sh +./scripts/prepare_image.sh -i base -s common + DB_HOST_IP=$(get_docker_db_ip_address) # Ensure permissions are set for everything within the test data directory. @@ -577,7 +579,7 @@ for image in $worker_images; do --tty \ --volume "$volume_directory":/home/user/data_store \ $INTERACTIVE \ - "$image_name" \ + "$image_name:$SYSTEM_VERSION" \ bash -c "$test_command" fi done