regression-test-docker-image.yml

name: "Regression Test - Docker Image"
on:
  workflow_dispatch:
  push:
    paths:
      - '**regression-test-docker-image.yml'
      - '**regression-test-standard-files.yml'
      - '**.py'
      - 'Dockerfile'
      - '**software-rules.json'
env:
  TEST_TAG: webperfse/webperf-core:test
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Setup QEMU # used to allow multiplatform docker builds
      uses: docker/setup-qemu-action@v3
    - name: Setup Docker Buildx # used to allow multiplatform docker builds
      uses: docker/setup-buildx-action@v3
    # - name: Login to Docker Hub
    #   uses: docker/login-action@v3
    #   with:
    #     username: ${{ secrets.DOCKERHUB_USERNAME }}
    #     password: ${{ secrets.DOCKERHUB_TOKEN }}
    - name: Build the Docker image # used to allow multiplatform docker builds
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64
        load: true
        tags: ${{ env.TEST_TAG }}
    - name: Check if we have correct python version
      run: |
        version=$(docker run ${{ env.TEST_TAG }} python --version)
        echo "Python version: $version"
        if [[ "$version" != "Python 3.13"* ]]; then
          echo "Error: Default python version is not 3.13 based, was we unable to change default version?"
          exit 1
        fi
    - name: Check if webperf_core help command works
      run: |
        help=$(docker run ${{ env.TEST_TAG }} python default.py -h)
        echo "$help"
        if [[ "$help" != *"-u/--url"* ]]; then
          echo "Error: WebPerf-core help command is NOT returning expected content"
          exit 1
        fi
    - name: Check if dependency tool works
      run: |
        help=$(docker run ${{ env.TEST_TAG }} python default.py --dependency)
        echo "$help"
        if [[ "$help" != *"Python: OK"* ]]; then
          echo "Error: dependency tool - Error with Python"
          exit 1
        fi
        if [[ "$help" != *"Node: OK"* ]]; then
          echo "Error: dependency tool - Error with Node"
          exit 1
        fi
        if [[ "$help" != *"Java: OK"* ]]; then
          echo "Error: dependency tool - Error with Java"
          exit 1
        fi
        if [[ "$help" != *"chrome: OK"* ]]; then
          echo "Error: dependency tool - Error with Chrome"
          exit 1
        fi
        if [[ "$help" != *"firefox: OK"* ]]; then
          echo "Error: dependency tool - Error with Firefox"
          exit 1
        fi
    - name: Check if webperf_core can run Test 1
      if: always()
      run: |
        testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 1 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 1 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 1 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
        if [[ "$testresult" != *"Performance:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 2
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 2 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Standards:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 4
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
        if [[ "$testresult" != *"Standards:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 5
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 6
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 6 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Standards:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 7
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 7 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Standards:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 9
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 9 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 10
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
        if [[ "$testresult" != *"A11y:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 15
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 15 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Performance:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 17
      if: always()
      run: |
        testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 17 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 18
      if: always()
      run: |
        testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 18 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"A11y:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 20
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 20 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 21
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 21 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 22
      if: always()
      run: |
        testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 22 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 22 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then
          sleep 5
          testresult=$(docker run --shm-size=4g -e MAX_OLD_SPACE_SIZE=3000 ${{ env.TEST_TAG }} python default.py -t 22 -r -i defaults/sites.json)
          echo "$testresult"
        fi
        if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
        if [[ "$testresult" != *"Overall:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 23
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 23 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 24
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 24 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 25
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 25 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Integrity & Security:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 26
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 26 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"A11y:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi
    - name: Check if webperf_core can run Test 27
      if: always()
      run: |
        testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 27 -r -i defaults/sites.json)
        echo "$testresult"
        if [[ "$testresult" != *"Standards:"* ]]; then
          echo "Error: WebPerf-core testresult is NOT returning expected review"
          exit 1
        fi