Rollback tests (#1054)

PR link: #1054
marqo-ai · Nov 28, 2024 · 05c3a22 · 05c3a22
1 parent 404485b
commit 05c3a22
Show file tree

Hide file tree

Showing 8 changed files with 438 additions and 280 deletions.
diff --git a/.github/workflows/backwards_compatibility_marqo_execution.yml b/.github/workflows/backwards_compatibility_marqo_execution.yml
@@ -5,7 +5,7 @@ on:
   workflow_call:
     # from_version: Used as: the identifier for a workflow call, for logging purposes and for pulling image from DockerHub. We need to pass a version here: ex: 2.11.1
     # to_version: Used as: the identifier for a workflow call and for logging purposes. We cannot use this to pull images from ECR or DockerHub (as opposed to from_version) since the to_version image has not been released yet. We need to pass a version here: ex: 2.11.5
-    # to_version_image_identifier: A unique identifier of the to_version image uploaded to ECR. Can either be the tag or the digest of the "To be released" image. This is specifically used to pull images from ECR.  We need to pass a full qualified docker image name with tag or digest here, example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef resp.
+    # to_image: A unique identifier of the to_version image uploaded to ECR. Can either be the tag or the digest of the "To be released" image. This is specifically used to pull images from ECR.  We need to pass a full qualified docker image name with tag or digest here, example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef resp.
     inputs:
       from_version:
         description: 'Source Marqo version. This is calculated in backwards_compatibility_marqo_orchestrator.yml and passed to this workflow'
@@ -15,14 +15,18 @@ on:
         description: 'Target Marqo version. This is used for logging purposes, to identify the target version of Marqo being tested and to calculate the from_versions in the backwards_compatibility_marqo_orchestrator.yml. It is NOT used to pull images from ECR.'
         required: true
         type: string
-      to_version_image_identifier:
+      to_image:
         description: 'To version image identifier is a unique identifier for the target Marqo image, which can either be a tag or a digest. It should contain complete qualified image name with tag or digest. For example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef. This is used to pull images from ECR.'
         required: true
         type: string
+      mode:
+        description: 'The mode in which the compatibility tests are to be run. Options: "backwards_compatibility", "rollback"'
+        required: true
+        type: string
   workflow_dispatch:
     # from_version: Used as: the identifier for a workflow call, for logging purposes and for pulling image from DockerHub. We need to pass a version here: ex: 2.11.1
     # to_version: Used as: the identifier for a workflow call and for logging purposes. We cannot use this to pull images from ECR or DockerHub (as opposed to from_version) since the to_version image has not been released yet. We need to pass a version here: ex: 2.11.5
-    # to_version_image_identifier: A unique identifier of the to_version image uploaded to ECR. Can either be the tag or the digest of the "To be released" image. This is specifically used to pull images from ECR.  We need to pass a full qualified docker image name with tag or digest here, example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef resp.
+    # to_image: A unique identifier of the to_version image uploaded to ECR. Can either be the tag or the digest of the "To be released" image. This is specifically used to pull images from ECR.  We need to pass a full qualified docker image name with tag or digest here, example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef resp.
 
     # If running manually, just specify the from_version, to_version and the fully qualified marqo image name with tag or digest in same format of the examples given above
     inputs:
@@ -34,10 +38,17 @@ on:
         description: 'Target Marqo version. This is used for logging purposes and to identify the target version of Marqo being tested.'
         required: true
         type: string
-      to_version_image_identifier:
+      to_image:
         description: 'To version image identifier is a unique identifier for the target Marqo image, which can either be a tag or a digest. It should contain complete qualified image name with tag or digest. For example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef. This is used to pull images from ECR.'
         required: true
         type: string
+      mode:
+        description: 'The mode in which the compatibility tests are to be run. Options: "backwards_compatibility", "rollback"'
+        options:
+          - backwards_compatibility
+          - rollback
+        required: true
+        type: choice
 
 jobs:
   Start-Runner:
@@ -79,7 +90,7 @@ jobs:
               {"Key": "PoloRole", "Value": "testing"}
             ]
             
-  backwards_compatibility:
+  compatibility_tests:
     # This job runs on the newly created runner
     runs-on: ${{ needs.start-runner.outputs.label }}
     needs: Start-Runner
@@ -115,19 +126,21 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      # Step to run the backwards compatibility test
-      - name: Run backwards_compatibility test
+      # Step to run the compatibility test. This step can run both backwards_compatibility and rollback tests, based on the MODE argument
+      - name: Run backwards compatibility test
+        id: run-backwards-compatibility
         env:
           FROM_VERSION: ${{ inputs.from_version || github.event.inputs.from_version }}
           TO_VERSION: ${{ inputs.to_version || github.event.inputs.to_version }}
-          TO_VERSION_IMAGE_IDENTIFIER: ${{ inputs.to_version_image_identifier || github.event.inputs.to_version_image_identifier }}
+          TO_IMAGE: ${{ inputs.to_image || github.event.inputs.to_image }}
+          MODE: ${{ inputs.mode || github.event.inputs.mode }}
         run: |
-              export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
-              python tests/backwards_compatibility_tests/compatibility_test_runner.py \
-              --mode=backwards_compatibility \
-              --from_version "$FROM_VERSION" \
-              --to_version "$TO_VERSION" \
-              --to_version_image_identifier "$TO_VERSION_IMAGE_IDENTIFIER" \
+            export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
+            python tests/backwards_compatibility_tests/compatibility_test_runner.py \
+            --mode "$MODE" \
+            --from_version "$FROM_VERSION" \
+            --to_version "$TO_VERSION" \
+            --to_image "$TO_IMAGE" \
 
   Stop-Runner:
     name: Stop self-hosted EC2 runner
@@ -139,7 +152,7 @@ jobs:
       statuses: write # Used implicitly by GitHub Actions to report job statuses and create check runs
     needs:
       - Start-Runner # required to get output from the start-runner job
-      - backwards_compatibility # required to wait when the main job is done
+      - compatibility_tests # required to wait when the main job is done
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:

diff --git a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml
@@ -134,9 +134,9 @@ jobs:
           aws-secret-access-key: ${{ secrets.ECR_PUSHER_AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-  run-execution-workflow:
-    # Job to trigger execution workflows for each version combination
-    name: Run all execution workflows
+  run-backwards-compatibility-execution-workflow:
+    # Job to trigger execution workflows for backwards compatibility test for each version combination
+    name: Run all backwards compatibility tests execution workflows
     needs: [orchestrate, check-if-image-exists, build-and-push-image]
     if: always() && (needs.orchestrate.result == 'success')
     strategy:
@@ -156,5 +156,33 @@ jobs:
       # Pass the image_identifier to the execution workflow. By image_identifier, we refer to the
       # complete qualified image name with the image digest (i.e 424082663841.dkr.ecr.us-east-1.amazonaws.com/marqo-compatibility-tests@sha256:1234567890abcdef).
       # The image_identifier can either come from the check-if-image-exists (i.e in case the image already exists in ECR) job or the build-and-push-image (i.e in case the image was built and pushed to ECR) job.
-      to_version_image_identifier: ${{ needs.check-if-image-exists.outputs.image_exists == 'true' && needs.check-if-image-exists.outputs.image_identifier 
-                                  || needs.build-and-push-image.outputs.image_identifier }}
+      to_image: ${{ needs.check-if-image-exists.outputs.image_exists == 'true' && needs.check-if-image-exists.outputs.image_identifier 
+                                  || needs.build-and-push-image.outputs.image_identifier }}
+      mode: "backwards_compatibility"
+
+
+  run-rollback-tests-execution-workflow:
+    # Job to trigger execution workflows for rollback test for each version combination
+    name: Run all rollback tests execution workflows
+    needs: [ orchestrate, check-if-image-exists, build-and-push-image ]
+    if: always() && (needs.orchestrate.result == 'success')
+    strategy:
+      matrix:
+        from_version: ${{ fromJson(needs.orchestrate.outputs.list) }}
+    uses: ./.github/workflows/backwards_compatibility_marqo_execution.yml
+    secrets: inherit
+    permissions:
+      contents: read # This permission is necessary to read repository contents
+      actions: write # Used by machulav/ec2-github-runner@v2 for managing self-hosted runners. The workflow needs to create and manage GitHub Actions runners on EC2
+      id-token: write # Used by aws-actions/configure-aws-credentials@v4. Required for AWS authentication and OIDC token management
+      checks: write # Used implicitly by GitHub Actions to report job statuses and create check runs
+      statuses: write # Used implicitly by GitHub Actions to report job statuses and create check runs
+    with:
+      from_version: ${{ matrix.from_version }}
+      to_version: ${{ needs.orchestrate.outputs.to_version }}
+      # Pass the image_identifier to the execution workflow. By image_identifier, we refer to the
+      # complete qualified image name with the image digest (i.e 424082663841.dkr.ecr.us-east-1.amazonaws.com/marqo-compatibility-tests@sha256:1234567890abcdef).
+      # The image_identifier can either come from the check-if-image-exists (i.e in case the image already exists in ECR) job or the build-and-push-image (i.e in case the image was built and pushed to ECR) job.
+      to_image: ${{ needs.check-if-image-exists.outputs.image_exists == 'true' && needs.check-if-image-exists.outputs.image_identifier
+        || needs.build-and-push-image.outputs.image_identifier }}
+      mode: "rollback"
diff --git a/tests/backwards_compatibility_tests/compatibility_test_logger.py b/tests/backwards_compatibility_tests/compatibility_test_logger.py
@@ -1,7 +1,7 @@
 import logging
 
 def get_logger(name):
-    logging.basicConfig(format='%(asctime)s | %(levelname)s | %(filename)s:%(lineno)d | %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
+    logging.basicConfig(format='%(asctime)s | %(levelname)s | %(filename)s:%(lineno)d | %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO)
     logger = logging.getLogger(name)
 
     return logger