pre-main.yaml

name: Test Incoming Changes

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:

permissions:
  contents: read

env:
  REGISTRY: quay.io
  REGISTRY_LOCAL: localhost
  TNF_IMAGE_NAME: testnetworkfunction/cnf-certification-test
  TNF_IMAGE_TAG: unstable
  OCT_IMAGE_NAME: testnetworkfunction/oct
  OCT_IMAGE_TAG: latest
  TNF_CONTAINER_CLIENT: docker
  TNF_NON_INTRUSIVE_ONLY: false
  TNF_ALLOW_PREFLIGHT_INSECURE: false
  TNF_DISABLE_CONFIG_AUTODISCOVER: false
  TNF_CONFIG_DIR: /tmp/tnf/config
  TNF_OUTPUT_DIR: /tmp/tnf/output
  TNF_SRC_URL: 'https://github.com/${{ github.repository }}'
  TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${TNF_IMAGE_NAME}:${TNF_IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}'
  TNF_SMOKE_TESTS_LOG_LEVEL: debug
  ON_DEMAND_DEBUG_PODS: false
  TERM: xterm-color
  CM_BIN: /usr/local/bin/checkmake
  CM_URL_LINUX: https://github.com/mrtazz/checkmake/releases/download/0.2.2/checkmake-0.2.2.linux.amd64 # yamllint disable-line
  SMOKE_TESTS_LABELS_FILTER: all
  SKIP_PRELOAD_IMAGES: true

concurrency:
  group: ${{ github.workflow }}-${{ github.event.number || github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    name: Run Linters and Vet
    runs-on: ubuntu-latest
    env:
      SHELL: /bin/bash

    steps:
      - name: Set up Go 1.22
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: 1.22.2

      - name: Disable default go problem matcher
        run: echo "::remove-matcher owner=go::"

      - name: Check out code into the Go module directory
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ github.sha }}

      - name: Install yaml dependency
        run: pip3 install --require-hashes -r .github/workflows/frozen-requirements-pre-main.txt
        env:
          SHELL: /bin/bash

      - name: Extract dependent Pull Requests
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install checkmake
        run: |
          curl --location --output $CM_BIN --silent $CM_URL_LINUX
          chmod +x $CM_BIN

      - name: Install Shfmt
        uses: mfinelli/setup-shfmt@031e887e39d899d773a7e9b6dd6472c2c23ff50d # v3.0.1

      - name: Golangci-lint
        uses: golangci/golangci-lint-action@9d1e0624a798bb64f6c3cea93db47765312263dc # v5.1.0
        with:
          version: v1.56
          args: --timeout 10m0s

      - name: Checkmake
        run: checkmake --config=.checkmake Makefile

      - name: Hadolint
        uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0
        with:
          dockerfile: Dockerfile
          recursive: true

      - name: Shfmt
        run: shfmt -d *.sh script

      - name: Markdownlint
        uses: nosborn/github-action-markdown-cli@9b5e871c11cc0649c5ac2526af22e23525fa344d # v3.3.0
        with:
          files: .

      - name: ShellCheck
        uses: ludeeus/action-shellcheck@cd81f4475ab741e097ec0fe73b692f3e49d66b8c # master

      # - name: Typos
      #   uses: crate-ci/typos@master

      - name: Yamllint
        uses: ibiqlik/action-yamllint@2576378a8e339169678f9939646ee3ee325e845c # v3.1.1
        with:
          config_file: .yamllint.yml

      - name: Go vet
        run: make vet
  unit-tests:
    name: Run Unit Tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-22.04]
    env:
      SHELL: /bin/bash

    steps:
      - name: Set up Go 1.22
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: 1.22.2

      - name: Disable default go problem matcher
        run: echo "::remove-matcher owner=go::"

      - name: Check out code into the Go module directory
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ github.sha }}

      - name: Install yaml dependency
        run: pip3 install --require-hashes -r .github/workflows/frozen-requirements-pre-main.txt
        env:
          SHELL: /bin/bash

      - name: Extract dependent Pull Requests
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Run Tests
        run: make test
        env:
          SHELL: /bin/bash

      - name: Quality Gate - Test coverage shall be above threshold
        env:
          TESTCOVERAGE_THRESHOLD: 15
        run: |
          echo "Quality Gate: checking test coverage is above threshold ..."
          echo "Threshold             : $TESTCOVERAGE_THRESHOLD %"
          totalCoverage=`UNIT_TEST='true' cat cover.out.tmp | grep -v "_moq.go" > cover.out; go tool cover -func=cover.out | grep total | grep -Eo '[0-9]+\.[0-9]+'`
          echo "Current test coverage : $totalCoverage %"
          if (( $(echo "$totalCoverage $TESTCOVERAGE_THRESHOLD" | awk '{print ($1 > $2)}') )); then
              echo OK
          else
              echo "Current test coverage is below threshold. Please add more unit tests or adjust threshold to a lower value."
              echo "Failed"
              exit 1
          fi

  smoke-tests-local:
    name: Run Local Smoke Tests
    runs-on: ubuntu-22.04
    env:
      SHELL: /bin/bash
      KUBECONFIG: '/home/runner/.kube/config'
      PFLT_DOCKERCONFIG: '/home/runner/.docker/config'

    steps:
      - name: Write temporary docker file
        run: |
          mkdir -p /home/runner/.docker
          touch ${PFLT_DOCKERCONFIG}
          echo '{ "auths": {} }' >> ${PFLT_DOCKERCONFIG}

      - name: Set up Go 1.22
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: 1.22.2

      - name: Disable default go problem matcher
        run: echo "::remove-matcher owner=go::"

      - name: Check out code into the Go module directory
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ github.sha }}

      - name: Install yaml dependency
        run: pip3 install --require-hashes -r .github/workflows/frozen-requirements-pre-main.txt
        env:
          SHELL: /bin/bash

      - name: Extract dependent Pull Requests
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      # Update the CNF containers, helm charts and operators DB.
      - name: Update the CNF DB
        run: |
          mkdir -p "${GITHUB_WORKSPACE}"/offline-db
          docker run \
            --env OCT_DUMP_ONLY=true \
            --rm \
            --volume "${GITHUB_WORKSPACE}"/offline-db:/tmp/dump:Z \
            ${REGISTRY}/${OCT_IMAGE_NAME}:${OCT_IMAGE_TAG}
          docker system prune --volumes -f

      - name: Build CNF test suite binary
        run: make build-cnf-tests

      - name: Remove go mod cache to save disk space.
        run: |
          df -h
          go clean -modcache || true
          df -h

      # Create a Kind cluster for testing.
      - name: Check out `cnf-certification-test-partner`
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          repository: test-network-function/cnf-certification-test-partner
          path: cnf-certification-test-partner

      - name: Bootstrap cluster, docker, and python
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make bootstrap-cluster && make bootstrap-docker-ubuntu-local && make bootstrap-python-ubuntu-local && pip3 install pyyaml

      # Restart docker using /mnt/docker-storage (sdb) instead of /var/lib/docker (sda).
      # This step needs to be done right after the partner repo's bootstrap scripts, as they
      # overwrite the docker's daemon.json.
      - name: Make docker to use /mnt (sdb) for storage
        run: |
          df -h
          lsblk
          sudo mkdir /mnt/docker-storage
          sudo jq '.  +={"data-root" : "/mnt/docker-storage"}' < /etc/docker/daemon.json > /tmp/docker-daemon.json
          sudo cp /tmp/docker-daemon.json /etc/docker/daemon.json
          cat /etc/docker/daemon.json
          sudo systemctl restart docker
          sudo ls -la /mnt/docker-storage

      - name: Run 'make rebuild-cluster'
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make rebuild-cluster

      - name: Run 'make install'
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make install

      # Perform smoke tests.
      - name: 'Test: Run test suites'
        run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} ./run-cnf-suites.sh -l "${SMOKE_TESTS_LABELS_FILTER}"

      - name: Upload smoke test results as an artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        if: always()
        with:
          name: smoke-tests
          path: |
            cnf-certification-test/*.tar.gz

      - name: Remove tarball(s) to save disk space.
        run: rm -f cnf-certification-test/*.tar.gz

      - name: Build the TNF tool
        run: make build-tnf-tool

      - name: Check the smoke test results against the expected results template
        run: make check-results

      - name: 'Test: Run preflight specific test suite'
        run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} ./run-cnf-suites.sh -l "preflight"

      - name: Upload preflight smoke test results as an artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        if: always()
        with:
          name: preflight-smoke-tests
          path: |
            cnf-certification-test/*.tar.gz

      - name: Remove tarball(s) to save disk space
        run: rm -f cnf-certification-test/*.tar.gz

  smoke-tests-container:
    name: Run Container Smoke Tests
    runs-on: ubuntu-22.04
    env:
      SHELL: /bin/bash
      KUBECONFIG: '/home/runner/.kube/config'
      PFLT_DOCKERCONFIG: '/home/runner/.docker/config'

    steps:
      - name: Write temporary docker file
        run: |
          mkdir -p /home/runner/.docker
          touch ${PFLT_DOCKERCONFIG}
          echo '{ "auths": {} }' >> ${PFLT_DOCKERCONFIG}

      # Create a Kind cluster for testing.
      - name: Check out `cnf-certification-test-partner`
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          repository: test-network-function/cnf-certification-test-partner
          path: cnf-certification-test-partner

      - name: Bootstrap cluster, docker, and python
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make bootstrap-cluster && make bootstrap-docker-ubuntu-local && make bootstrap-python-ubuntu-local && pip3 install pyyaml

      # Restart docker using /mnt/docker-storage (sdb) instead of /var/lib/docker (sda).
      # This step needs to be done right after the partner repo's bootstrap scripts, as they
      # overwrite the docker's daemon.json.
      - name: Make docker to use /mnt (sdb) for storage
        run: |
          df -h
          lsblk
          sudo mkdir /mnt/docker-storage
          sudo jq '.  +={"data-root" : "/mnt/docker-storage"}' < /etc/docker/daemon.json > /tmp/docker-daemon.json
          sudo cp /tmp/docker-daemon.json /etc/docker/daemon.json
          cat /etc/docker/daemon.json
          sudo systemctl restart docker
          sudo ls -la /mnt/docker-storage

      - name: Run 'make rebuild-cluster'
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make rebuild-cluster

      - name: Run 'make install'
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: cd ${GITHUB_WORKSPACE}/cnf-certification-test-partner && make install

      # needed by depends-on-action
      - name: Set up Go 1.22
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
        with:
          go-version: 1.22.2

      # Perform smoke tests using a TNF container.
      - name: Check out code into the Go module directory
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ github.sha }}

      - name: Extract dependent Pull Requests
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Build the `cnf-certification-test` image
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 90
          max_attempts: 3
          command: make build-image-local
        env:
          IMAGE_TAG: ${TNF_IMAGE_TAG}

      # Prepare collector to be used when running smoke tests
      - name: Check out `Collector`
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          repository: test-network-function/collector
          path: collector

      # Clean up unused container image layers. We need to filter out a possible error return code
      # from docker with "|| true" as some images might still be used by running kind containers and
      # won't be removed.
      - name: Remove unnamed/dangling container images to save space. Show disk space before and after removing them.
        run: |
          df -h
          docker rmi $(docker images -f "dangling=true" -q) || true
          df -h

      - name: Create required TNF config files and directories
        run: |
          mkdir -p $TNF_CONFIG_DIR $TNF_OUTPUT_DIR
          cp cnf-certification-test/*.yml $TNF_CONFIG_DIR
        shell: bash

      - name: Get Collector's CI credentials
        run: |
          echo "collector_ciuser=ciuser_${{ github.run_id }}" >> $GITHUB_OUTPUT
          echo "collector_cipassword=cipassword" >> $GITHUB_OUTPUT
        id: set_collector_ci_creds

      - name: Update Collector's CI credentials and Print username
        run: |
          echo Collector CI username: ${{ steps.set_collector_ci_creds.outputs.collector_ciuser }}
          echo "COLLECTOR_CIUSER=${{ steps.set_collector_ci_creds.outputs.collector_ciuser }}" >> $GITHUB_ENV
          echo "COLLECTOR_CIPASSWORD=${{ steps.set_collector_ci_creds.outputs.collector_cipassword }}" >> $GITHUB_ENV

      - name: Ensure COLLECTOR_CIUSER and COLLECTOR_CIPASSWORD are set
        run: '[[ -n "$COLLECTOR_CIUSER" ]] && [[ -n "$COLLECTOR_CIPASSWORD" ]]'

      - name: Modify TNF config with CI collector credentials
        run: |
          sed -i\
            -e '/executedBy/s/""/"CI"/g' \
            -e '/partnerName/s/""/"${{ env.COLLECTOR_CIUSER }}"/g' \
            -e '/collectorAppPassword/s/""/"${{ env.COLLECTOR_CIPASSWORD }}"/g' \
            $TNF_CONFIG_DIR/tnf_config.yml

      - name: 'Test: Run without any TS, just get diagnostic information'
        run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }}

      - name: 'Test: Run Smoke Tests in a TNF container'
        run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} TNF_ENABLE_DATA_COLLECTION=true ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -l "${SMOKE_TESTS_LABELS_FILTER}"

      - name: Run sanity check on collector
        uses: ./collector/.github/actions/run-sanity-check
        with:
          working_directory: collector
          collector_username: ${COLLECTOR_CIUSER}
          collector_password: ${COLLECTOR_CIPASSWORD}

      - name: Upload container test results as an artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        if: always()
        with:
          name: smoke-tests-container
          path: |
            ${{ env.TNF_OUTPUT_DIR }}/*.tar.gz

      - name: Remove tarball(s) to save disk space.
        run: rm -f ${{ env.TNF_OUTPUT_DIR }}/*.tar.gz

      - name: Build the TNF tool
        run: make build-tnf-tool

      - name: Check the smoke test results against the expected results template
        run: ./tnf check results --log-file="${TNF_OUTPUT_DIR}"/cnf-certsuite.log

      - name: 'Test: Run Preflight Specific Smoke Tests in a TNF container'
        run: TNF_LOG_LEVEL=${TNF_SMOKE_TESTS_LOG_LEVEL} ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -l "preflight"

  # Only run this job if the previous jobs are successful that build the ARM and x86 images.
  create-manifest-multiarch:
    name: Create manifest list for multi-arch image
    needs: [unit-tests, smoke-tests-container]
    runs-on: ubuntu-22.04
    if: github.event_name != 'pull_request' && needs.smoke-tests-container.result == 'success' && needs.unit-tests.result == 'success'
    steps:
      - name: Check out code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          ref: ${{ github.sha }}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      - name: Login to Quay.io
        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
        if: ${{ github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.QUAY_ROBOT_USERNAME }}
          password: ${{ secrets.QUAY_ROBOT_TOKEN }}

      - name: Build and push the unstable images for multi-arch
        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
        if: ${{ github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            ${{ env.REGISTRY }}/${{ env.TNF_IMAGE_NAME }}:${{ env.TNF_IMAGE_TAG }}
      
      - name: (if on main and upstream) Send chat msg to dev team if failed to create container image.
        if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
        uses: ./.github/actions/slack-webhook-sender
        with:
          message: 'Failed to create the *unstable* container manifest'
          slack_webhook: '${{ secrets.SLACK_ALERT_WEBHOOK_URL }}'


  check-all-dependencies-are-merged:
    name: Check all the PR dependencies are merged
    runs-on: ubuntu-latest
    steps:

      - name: Checkout code
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - name: Check all dependent Pull Requests are merged
        uses: depends-on/depends-on-action@9e8a61fce18b15281e831f1bba0e14c71d1e1f46 # main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          check-unmerged-pr: true