Upgrade circleci config version and improve api/security tests

.circleci/config.yml

@@ -2,7 +2,41 @@ defaults: &defaults
     docker:
         - image: circleci/node:15.2.1-browsers
 
-version: 2
+version: 2.1
+commands:
+  build_push_image:
+    parameters:
+      push:
+        description: Push image to DockerHub
+        type: string
+        default: "false"
+    steps:
+      - run:
+          name: Build cBioPortal docker image
+          environment:
+            DOCKER_REPO: cbioportal/cbioportal-dev
+          command: |
+            export DOCKER_TAG=$CIRCLE_SHA1
+            URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
+            TAG_FOUND=$(curl -s $URL | jq -r .name)
+            if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
+              echo "Image already exists. Skipping build step!"
+              exit 0
+            fi
+            cd cbioportal-test
+            ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=<<parameters.push>> --skip_web_and_data=true
+            if [ "<<parameters.push>>" = "false" ]; then
+              EXISTS=$(docker inspect --type=image $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?);
+            else
+              EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?);
+            fi
+            if [ $EXISTS -eq 0 ]; then
+              echo "Build succeeded!"
+            else
+              echo "Build failed!"
+              exit 1
+            fi
+
 jobs:
     build_backend:
         docker:
@@ -258,53 +292,71 @@ jobs:
             paths:
               - cbioportal-frontend
 
-    build_push_image:
+    checkout_pr:
       machine:
         image: ubuntu-2204:2024.08.1
       resource_class: medium
       working_directory: /tmp/repos
       steps:
-        - attach_workspace:
-            at: /tmp/repos
         - checkout:
             path: /tmp/repos/cbioportal
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - cbioportal
+
+    build_image:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      environment:
+        DOCKER_REPO: cbioportal/cbioportal-dev
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - build_push_image:
+            push: "false"
         - run:
-            name: Build cBioPortal docker image
-            environment:
-              DOCKER_REPO: cbioportal/cbioportal-dev
+            name: Save cbioportal image as tar
             command: |
               export DOCKER_TAG=$CIRCLE_SHA1
-              URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
-              TAG_FOUND=$(curl -s $URL | jq -r .name)
-              if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
-                echo "Image already exists. Skipping build step!"
-                exit 0
-              fi
-              cd cbioportal-test
-              ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=true --skip_web_and_data=true
-              EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?)
-              if [ $EXISTS -eq 0 ]; then
-                echo "Build succeeded!"
-              else
-                echo "Build failed!"
-                exit 1
-              fi
+              docker save -o $DOCKER_TAG-web-shenandoah.tar $DOCKER_REPO:$DOCKER_TAG-web-shenandoah
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - "*.tar"
+
+    push_image:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - build_push_image:
+            push: "true"
 
     run_api_tests:
       machine:
         image: ubuntu-2204:2024.08.1
         docker_layer_caching: true
       resource_class: large
       working_directory: /tmp/repos
+      environment:
+        DOCKER_REPO: cbioportal/cbioportal-dev
       steps:
         - attach_workspace:
             at: /tmp/repos
-        - checkout:
-            path: /tmp/repos/cbioportal
+        - run:
+            name: Load cbioportal image
+            command: |
+              export DOCKER_TAG=$CIRCLE_SHA1
+              docker load -i $DOCKER_TAG-web-shenandoah.tar
         - run:
             name: Instantiate a cbioportal instance
             environment:
-              DOCKER_REPO: cbioportal/cbioportal-dev
               APP_CLICKHOUSE_MODE: "true"
             command: |
               cd cbioportal-test
@@ -346,8 +398,6 @@ jobs:
               yarn --ignore-engines
               yarn run apitests
 
-        - store_artifacts:
-            path: /tmp/repos/cbioportal-test/web-metadata.json
         - store_artifacts:
             path: /tmp/repos/docker-compose-logs.txt
 
@@ -356,43 +406,43 @@ jobs:
         image: ubuntu-2204:2024.08.1
         docker_layer_caching: true
       resource_class: medium
+      working_directory: /tmp/repos
       environment:
         BASE_REPO: cbioportal/cbioportal
         DEV_REPO: cbioportal/cbioportal-dev
+        OUTPUT_FORMAT: '{severity: .cvss.severity, source_id: .source_id, vulnerable_range: .vulnerable_range, fixed_by: .fixed_by, url: .url, description: .description}'
+        SORT: 'sort_by(.severity | if . == "CRITICAL" then 0 elif . == "HIGH" then 1 elif . == "MEDIUM" then 2 elif . == "LOW" then 3 else 4 end)'
       steps:
+        - attach_workspace:
+              at: /tmp/repos
         - run:
             name: Install Docker Scout
             command: |
               curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s -- -b /home/circleci/bin
         - run:
             name: Log in to Docker
             command: |
-              echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin;
+              echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        - run:
+            name: Load cbioportal image
+            command: |
+              export DOCKER_TAG=$CIRCLE_SHA1
+              docker load -i $DOCKER_TAG-web-shenandoah.tar
+        - run:
+            name: Run Docker Scout on master
+            command: |
+              IMAGE=$BASE_REPO:master-web-shenandoah
+              docker pull $IMAGE
+              docker-scout cves $IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > master_report.sbom
         - run:
-            name: Wait for cbioportal docker images
+            name: Run Docker Scout on PR
             command: |
-              URL="https://hub.docker.com/v2/repositories/$DEV_REPO/tags/$CIRCLE_SHA1-web-shenandoah"
-              while true; do
-                TAG_FOUND=$(curl -s $URL | jq -r .name)
-                if [ $TAG_FOUND = "$CIRCLE_SHA1-web-shenandoah" ]; then
-                  echo "Image found!"
-                  exit 0
-                fi
-                echo "Image not found yet. Waiting for API Tests to finish building. Retrying in 30 seconds..."
-                sleep 30
-              done
+              IMAGE=$DEV_REPO:$CIRCLE_SHA1-web-shenandoah
+              docker-scout cves $IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > pr_report.sbom
         - run:
-            name: Run Docker Scout vulnerability test
+            name: Analyze and report results
             command: |
-              BASE_IMAGE=$BASE_REPO:master-web-shenandoah
-              PR_IMAGE=$DEV_REPO:$CIRCLE_SHA1-web-shenandoah
-              OUTPUT_FORMAT='{severity: .cvss.severity, source_id: .source_id, vulnerable_range: .vulnerable_range, fixed_by: .fixed_by, url: .url, description: .description}'
-              SORT='sort_by(.severity | if . == "CRITICAL" then 0 elif . == "HIGH" then 1 elif . == "MEDIUM" then 2 elif . == "LOW" then 3 else 4 end)'
-              docker pull $BASE_IMAGE
-              docker pull $PR_IMAGE
-              docker-scout cves $BASE_IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > base_report.sbom
-              docker-scout cves $PR_IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > pr_report.sbom
-              DIFF=$(jq -s 'map(map(.source_id)) | .[0] - .[1]' pr_report.sbom base_report.sbom)
+              DIFF=$(jq -s 'map(map(.source_id)) | .[0] - .[1]' pr_report.sbom master_report.sbom)
               COUNT=$(echo $DIFF | jq 'length')
               if [ "$COUNT" -gt 0 ]; then
                 printf "New vulnerabilities found: $COUNT\n"
@@ -404,7 +454,6 @@ jobs:
               fi
 
 workflows:
-    version: 2
     end_to_end_tests:
         jobs:
             - build_backend
@@ -417,28 +466,29 @@ workflows:
                     - build_backend
                     - pull_frontend_codebase
                     - install_yarn
-    api_tests:
+    tests:
       jobs:
+        - checkout_pr
         - pull_cbioportal_test_codebase
         - pull_cbioportal_frontend_codebase
-        - wait_for_approval:
-            type: approval
+        - build_image:
             requires:
+              - checkout_pr
               - pull_cbioportal_test_codebase
-              - pull_cbioportal_frontend_codebase
-        - build_push_image:
+        - push_image:
             context:
               - api-tests
             requires:
-              - wait_for_approval
+              - checkout_pr
+              - pull_cbioportal_test_codebase
         - run_api_tests:
             context:
               - api-tests
             requires:
-              - build_push_image
-
-    security_tests:
-      jobs:
+              - build_image
+              - pull_cbioportal_frontend_codebase
         - run_security_tests:
             context:
-              - docker-scout
+              - docker-scout
+            requires:
+              - build_image