RP E2E Test Execution

.pipelines/ci.yml

@@ -1,4 +1,4 @@
-# Azure DevOps Pipeline running CI
+# Azure DevOps Pipeline running CI and E2E
 
 trigger:
   branches:
@@ -13,70 +13,201 @@ trigger:
 
 variables:
   - template: vars.yml
+  - name: CI
+    value: true
   - name: REGISTRY
     value: registry.access.redhat.com
   - name: LOCAL_ARO_RP_IMAGE
-    value: "arosvcdev.azurecr.io/aro"
+    value: arosvcdev.azurecr.io/aro
   - name: LOCAL_ARO_AZEXT_IMAGE
-    value: "arosvcdev.azurecr.io/azext-aro"
+    value: arosvcdev.azurecr.io/azext-aro
   - name: LOCAL_VPN_IMAGE
-    value: "arosvcdev.azurecr.io/vpn"
+    value: arosvcdev.azurecr.io/vpn
+  - name: LOCAL_E2E_IMAGE
+    value: arosvcdev.azurecr.io/e2e
   - name: TAG
     value: $(Build.BuildId)
   - name: VERSION
     value: $(Build.BuildId)
+  - name: ARO_IMAGE
+    value: arosvcdev.azurecr.io/aro:$(Build.BuildId)
+  - name: ARO_SELENIUM_HOSTNAME
+    value: localhost
+  - name: E2E_LABEL
+    value: "!smoke&&!regressiontest"
 
-jobs:
-  - job: Build_Test_And_Push_Az_ARO_Extension
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test the Az ARO Extension
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
-        displayName: 🛠 Build & Test Az ARO Extension
-
-      # Push the image to ACR
-      - template: ./templates/template-acr-push.yml
-        parameters:
-          acrFQDN: 'arosvcdev.azurecr.io'
-          repository: 'azext-aro'
-          pushLatest: true
-
-  - job: Build_And_Test_RP_And_Portal
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test RP and Portal
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
-        displayName: 🛠 Build & Test RP and Portal
-
-      # Publish test results
-      - task: PublishTestResults@2
-        displayName: 📊 Publish tests results
-        inputs:
-          testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
-        condition: succeededOrFailed()
-
-      # Publish code coverage results
-      - task: PublishCodeCoverageResults@2
-        displayName: 📈 Publish code coverage
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
-          failIfCoverageEmpty: false
-        condition: succeededOrFailed()
-
-      # Push the image to ACR
-      - template: ./templates/template-acr-push.yml
-        parameters:
-          acrFQDN: 'arosvcdev.azurecr.io'
-          repository: 'aro'
-          pushLatest: true
+stages:
+  - stage: Containerized_CI
+    jobs:
+      - job: Build_Test_And_Push_Az_ARO_Extension
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build and test the Az ARO Extension
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
+            displayName: 🛠 Build & Test Az ARO Extension
+
+          # Push the image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'azext-aro'
+              pushLatest: true
+
+      - job: Build_And_Test_RP_And_Portal
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build and test RP and Portal
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
+            displayName: 🛠 Build & Test RP and Portal
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: 📊 Publish tests results
+            inputs:
+              testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
+            condition: succeededOrFailed()
+
+          # Publish code coverage results
+          - task: PublishCodeCoverageResults@2
+            displayName: 📈 Publish code coverage
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
+              failIfCoverageEmpty: false
+            condition: succeededOrFailed()
+
+          # Push the RP image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'aro'
+              pushLatest: true
+
+      - job: Build_And_Push_E2E_Image
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build the E2E image
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make aro-e2e VERSION=$(VERSION)
+            displayName: 🛠 Build the E2E image
+
+          # Push the E2E image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'e2e'
+              pushLatest: true
+
+  - stage: E2E  # E2E Stage using Docker Compose
+    dependsOn: Containerized_CI
+    jobs:
+      - job: Run_E2E_Tests
+        timeoutInMinutes: 0
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          # Checkout the code
+          - template: ./templates/template-checkout.yml
+
+          # Install Docker, Docker Compose, and dependencies
+          - bash: |
+              . ./hack/e2e/utils.sh
+              install_docker_dependencies
+            displayName: Install Docker and Docker Compose
+
+          # AZ CLI Login
+          - template: ./templates/template-az-cli-login.yml
+            parameters:
+              azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
+
+          # Get Kubeconfig for AKS Cluster with corrected Key Vault configuration
+          - bash: |
+              az account set -s $AZURE_SUBSCRIPTION_ID
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . secrets/env
+              export KEYVAULT_PREFIX="e2e-classic-eastus-cls"
+
+              # Retrieve the kubeconfig
+              hack/get-admin-aks-kubeconfig.sh > aks.kubeconfig
+
+              if [ -f aks.kubeconfig ]; then
+                echo "Kubeconfig retrieved successfully."
+              else
+                echo "Failed to retrieve Kubeconfig."
+                exit 1
+              fi
+            displayName: Get Kubeconfig for AKS Cluster
+
+
+          # Run the E2E test suite
+          - bash: |
+              az account set -s $AZURE_SUBSCRIPTION_ID
+              az acr login --name arosvcdev
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . ./hack/e2e/run-rp-and-e2e.sh
+              deploy_e2e_db
+              register_sub
+              docker compose up e2e
+              # Check if the E2E tests failed
+              E2E_EXIT_CODE=$?
+              if [ $E2E_EXIT_CODE -ne 0 ]; then
+                echo "##vso[task.logissue type=error] E2E tests failed. Check the logs for more details."
+                exit 1
+              else
+                echo "E2E tests passed."
+              fi
+            displayName: ⚙️  Run E2E Test Suite
+
+          # Log the output from the services in case of failure
+          - bash: |
+              docker compose logs vpn
+              docker compose logs selenium
+              docker compose logs rp
+              docker compose logs portal
+              docker compose logs e2e
+            displayName: Log Service Output
+            condition: always()
+
+          # Collect must-gather logs
+          - bash: |
+              wget -nv https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/$(OpenShiftCLIVersion)/openshift-client-linux-$(OpenShiftCLIVersion).tar.gz
+              tar xf openshift-client-linux-$(OpenShiftCLIVersion).tar.gz
+              ./oc adm must-gather --image quay.io/cmarches/aro-must-gather:20231030.00
+              tar cf must-gather.tar.gz must-gather.local.*
+            displayName: Collect must-gather
+            condition: Failed()
+          # Publish the must-gather result to the pipeline
+          - publish: must-gather.tar.gz
+            artifact: must-gather
+            displayName: Append must-gather to Pipeline
+            condition: Failed()
+
+          # Clean up Docker Compose services
+          - bash: |
+              docker compose down
+              rm -f aks.kubeconfig
+            displayName: Cleanup Docker Compose Services and Kubeconfig
+            condition: always()
+
+          # Clean Up Database
+          - bash: |
+              az cosmosdb sql database delete --name "$DATABASE_NAME" --yes --account-name "$DATABASE_ACCOUNT_NAME" --resource-group "$RESOURCEGROUP"
+            displayName: Clean Up Database
+            condition: always()
+
+          # AZ CLI Logout
+          - template: ./templates/template-az-cli-logout.yml

.pipelines/templates/deploy-hive-operator.yml

@@ -0,0 +1,40 @@
+# Azure DevOps Pipeline for deploying Hive Operator
+trigger: none
+pr: none
+
+variables:
+  - template: vars.yml
+
+stages:
+  - stage: Deploy_Hive_Operator
+    jobs:
+      - job: Deploy_Hive_Operator
+        pool:
+          name: 1es-aro-ci-pool
+
+        steps:
+          # Checkout the code
+          - template: ./templates/template-checkout.yml
+
+          # AZ CLI Login
+          - template: ./templates/template-az-cli-login.yml
+            parameters:
+              azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
+
+          # Deploy Hive Operator
+          - bash: |
+              az account set -s $AZURE_SUBSCRIPTION_ID
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . secrets/env
+              docker compose -f docker-compose.yml up -d vpn
+              while [ "$(docker inspect --format '{{.State.Health.Status}}' vpn)" != "healthy" ]; do
+                echo "Waiting for VPN to be healthy..."
+                sleep 10
+              done
+              docker ps
+              kubectl get nodes
+              ./hack/hive/hive-dev-install.sh
+            displayName: Deploy Hive Operator
+
+          # AZ CLI Logout
+          - template: ./templates/template-az-cli-logout.yml

.pipelines/templates/template-az-cli-login.yml

@@ -7,7 +7,13 @@ steps:
       set -e
 
       trap 'rm -f devops-spn.json' EXIT
-      base64 -d >devops-spn.json <<<${{ parameters.azureDevOpsJSONSPN }}
+      echo "${{ parameters.azureDevOpsJSONSPN }}" | base64 -d > devops-spn.json
 
-      az login --service-principal -u "$(jq -r .clientId <devops-spn.json)" -p "$(jq -r .clientSecret <devops-spn.json)" -t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
+      az login --service-principal \
+        -u "$(jq -r .clientId <devops-spn.json)" \
+        -p "$(jq -r .clientSecret <devops-spn.json)" \
+        -t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
+
+      # Cleanup
+      rm -f devops-spn.json
     displayName: 🗝 AZ Login

Dockerfile.aro-e2e

@@ -17,6 +17,9 @@ FROM ${REGISTRY}/ubi8/ubi-minimal
 RUN microdnf update && microdnf clean all
 COPY --from=builder /root/go/bin/gojq /usr/local/bin/jq
 COPY --from=builder /app/aro /app/e2e.test /app/db /app/cluster /app/portalauth /usr/local/bin/
+# Setting ENV HOME=/tmp does not change the user’s default home directory of /
+# This setting is required to keep the existing e2e pipeline working without any code changes
+COPY --from=builder /app/portalauth /
 ENTRYPOINT ["aro"]
 EXPOSE 2222/tcp 8080/tcp 8443/tcp 8444/tcp 8445/tcp
 USER 1000

Dockerfile.ci-rp

@@ -72,6 +72,7 @@ FROM ${REGISTRY}/ubi8/ubi-minimal AS final
 LABEL aro-final=true
 RUN microdnf update && microdnf clean all
 COPY --from=builder /app/aro /app/e2e.test /usr/local/bin/
+COPY --from=builder /app/report.xml /app/coverage.xml /app/
 ENTRYPOINT ["aro"]
 EXPOSE 2222/tcp 8080/tcp 8443/tcp 8444/tcp
 USER 1000

Makefile

@@ -380,9 +380,8 @@ DOCKER_BUILD_CI_ARGS ?=
 # Image names that will be found in the local podman image registry after build
 # (tags are always VERSION).
 LOCAL_ARO_RP_IMAGE ?= aro
-LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build
-LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build
-LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro
+LOCAL_E2E_IMAGE ?= e2e
+LOCAL_ARO_AZEXT_IMAGE ?= azext-aro
 LOCAL_TUNNEL_IMAGE ?= aro-tunnel
 
 ###############################################################################
@@ -394,7 +393,7 @@ ci-azext-aro:
 		-f Dockerfile.ci-azext-aro \
 		--platform=linux/amd64 \
 		--no-cache=$(NO_CACHE) \
-		-t $(LOCAL_AZ_EXT_ARO_IMAGE):$(VERSION)
+		-t $(LOCAL_ARO_AZEXT_IMAGE):$(VERSION)
 
 .PHONY: ci-clean
 ci-clean:
@@ -410,17 +409,24 @@ ci-rp: fix-macos-vendor
 		--build-arg REGISTRY=${REGISTRY} \
 		--build-arg ARO_VERSION=${VERSION} \
 		--no-cache=${NO_CACHE} \
-		--target=builder \
-		-t ${LOCAL_ARO_RP_BUILD_IMAGE}:${VERSION}
-
-	docker compose build rp
+		-t ${LOCAL_ARO_RP_IMAGE}:${VERSION}
 
 	# Extract test coverage files from build to local filesystem
-	docker create --name extract_cover_out ${LOCAL_ARO_RP_BUILD_IMAGE}:${VERSION}; \
+	docker create --name extract_cover_out ${LOCAL_ARO_RP_IMAGE}:${VERSION}; \
 	docker cp extract_cover_out:/app/report.xml ./report.xml; \
 	docker cp extract_cover_out:/app/coverage.xml ./coverage.xml; \
 	docker rm extract_cover_out;
 
+.PHONY: aro-e2e
+aro-e2e: fix-macos-vendor
+	docker build . ${DOCKER_BUILD_CI_ARGS} \
+		-f Dockerfile.aro-e2e \
+		--ulimit=nofile=4096:4096 \
+		--build-arg REGISTRY=${REGISTRY} \
+		--build-arg ARO_VERSION=${VERSION} \
+		--no-cache=${NO_CACHE} \
+		-t ${LOCAL_E2E_IMAGE}:${VERSION}
+
 .PHONY: ci-tunnel
 ci-tunnel: fix-macos-vendor
 	podman $(PODMAN_REMOTE_ARGS) \