qe-hosted.yml

name: QE Testing (Ubuntu-hosted)

on:
  pull_request:
    branches: [ main ]
  workflow_dispatch:
  # Schedule a daily cron at midnight UTC
  schedule:
    - cron: '0 0 * * *'

permissions:
  contents: read

env:
  QE_REPO: test-network-function/cnfcert-tests-verification

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Build the image used for testing first, then pass the reference to the QE tests.
  # This saves time and resources by not building the image in each QE suite.
  build-image-for-qe:
    runs-on: ubuntu-24.04
    steps:
      - name: Check out code
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
        with:
          ref: ${{ github.sha }}

      - name: Setup docker buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      - name: Build temporary image tag for this PR
        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
        with:
          context: .
          file: ./Dockerfile
          tags: quay.io/testnetworkfunction/cnf-certification-test:localtest
          outputs: type=docker,dest=/tmp/testimage.tar
      
      - name: Store image as artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: testimage
          path: /tmp/testimage.tar

  qe-testing:
    runs-on: ubuntu-24.04
    needs: build-image-for-qe
    if: needs.build-image-for-qe.result == 'success'
    strategy:
      fail-fast: false
      matrix:
        suite: [accesscontrol, affiliatedcertification, manageability, networking, lifecycle, performance, platformalteration, observability, operator]
    env:
      SHELL: /bin/bash
      KUBECONFIG: '/home/runner/.kube/config'
      PFLT_DOCKERCONFIG: '/home/runner/.docker/config'
      TEST_TNF_IMAGE_NAME: quay.io/testnetworkfunction/cnf-certification-test
      TEST_TNF_IMAGE_TAG: localtest
      DOCKER_CONFIG_DIR: '/home/runner/.docker/'
      SKIP_PRELOAD_IMAGES: true

    steps:
      - name: Check out code
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
        with:
          ref: ${{ github.sha }}

      - name: Run initial setup
        uses: ./.github/actions/setup

      # Download the image from the artifact and load it into the docker daemon.
      - name: Setup docker buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      # Create a Kind cluster for testing.
      - name: Check out `cnf-certification-test-partner`
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
        with:
          repository: test-network-function/cnf-certification-test-partner
          path: cnf-certification-test-partner

      - name: Check if /etc/docker/daemon.json exists and if not, create it.
        run: |
          if [ ! -f /etc/docker/daemon.json ]; then
            echo '{}' | sudo tee /etc/docker/daemon.json
          fi

      - name: Bootstrap cluster and docker
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make bootstrap-cluster; make bootstrap-docker-ubuntu-local

      # Restart docker using /mnt/docker-storage (sdb) instead of /var/lib/docker (sda).
      # This step needs to be done right after the partner repo's bootstrap scripts, as they
      # overwrite the docker's daemon.json.
      - name: Make docker to use /mnt (sdb) for storage
        run: |
          df -h
          lsblk
          sudo mkdir /mnt/docker-storage
          sudo jq '.  +={"data-root" : "/mnt/docker-storage"}' < /etc/docker/daemon.json > /tmp/docker-daemon.json
          sudo cp /tmp/docker-daemon.json /etc/docker/daemon.json
          cat /etc/docker/daemon.json
          sudo systemctl restart docker
          sudo ls -la /mnt/docker-storage

      - name: Download image from artifact
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: testimage
          path: /tmp

      - name: Load image into docker
        run: docker load --input /tmp/testimage.tar

      - name: Run 'make rebuild-cluster'
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; make rebuild-cluster

      - name: Install partner resources
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner; python3 -m venv .venv; source .venv/bin/activate; pip install --upgrade pip; pip install jinjanator; cp .venv/bin/jinjanate .venv/bin/j2; make install-for-qe

      - name: Show pods
        run: oc get pods -A

      - name: Wait for all pods to be ready
        run: ./scripts/wait-for-all-pods-running.sh
        working-directory: cnf-certification-test-partner
        timeout-minutes: 10

      - name: Clone the QE repository
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
        with:
          repository: ${{ env.QE_REPO }}
          path: cnfcert-tests-verification
          ref: main

      - name: Extract dependent Pull Requests
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Run the tests
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 60
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} JOB_ID=${{github.run_id}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features

      - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
        if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
        env:
          COMMIT_SHA: ${{ github.sha }}
          JOB_RUN_ID: ${{ github.run_id }}
          JOB_RUN_ATTEMPT: ${{ github.run_attempt }}
          GITHUB_REPO: https://github.com/test-network-function/cnf-certification-test
        run: |
          curl -X POST --data "{
              \"text\": \"🚨⚠️  Failed to run Github hosted QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'