sycl-linux-run-tests.yml

name: SYCL E2E

on:
  workflow_call:
    inputs:
      name:
        type: string
        required: True

      runner:
        type: string
        required: True
      image:
        type: string
        required: True
      image_options:
        type: string
        required: True

      target_devices:
        type: string
        required: True
      extra_cmake_args:
        description: |
          If empty, then HIP_PLATFORM and AMD_ARCH would be automatically added
          if inputs.target_devices contains 'ext_oneapi_hip'
        type: string
        required: False
      tests_selector:
        description: |
          Three options: "e2e", "cts", "benchmark".
        type: string
        default: "e2e"

      extra_lit_opts:
        description: |
          Extra options to be added to LIT_OPTS.
        type: string
        default: ''

      ref:
        type: string
        required: True
      merge_ref:
        description: |
          Commit-ish to merge post-checkout if non-empty. Must be reachable from
          the default_branch input paramter.
        type: string
        default: 'FETCH_HEAD'
        required: False

      sycl_toolchain_artifact:
        type: string
        default: ''
        required: False
      sycl_toolchain_archive:
        type: string
        default: ''
        required: False
      sycl_toolchain_decompress_command:
        type: string
        default: ''
        required: False

      reset_gpu:
        type: string
        required: False
      install_drivers:
        type: string
        required: False
      use_dev_igc:
        type: string
        required: False
      env:
        type: string
        default: '{}'
        required: False

  workflow_dispatch:
    inputs:
      runner:
        type: choice
        options:
          - '["Linux", "gen12"]'
          - '["amdgpu"]'
          - '["Linux", "arc"]'
          - '["cts-cpu"]'
      image:
        description: |
          Use option ending with ":build" for AMDGPU, ":latest" for the rest.
        type: choice
        options:
          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest'
          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build'
      image_options:
        description: |
          Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest.
        type: choice
        options:
          - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN'
          - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN'
      target_devices:
        type: choice
        options:
          - 'opencl:cpu'
          - 'opencl:gpu'
          - 'opencl:fpga'
          - 'ext_oneapi_level_zero:gpu'
          - 'ext_oneapi_hip:gpu'
      tests_selector:
        type: choice
        options:
          - e2e
          - cts
          - benchmark

      env:
        description: |
          Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT.
          LIT_OPTS won't work as we redefine it as part of this workflow.

          For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to
          build.

          Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}'
        default: '{}'

      extra_lit_opts:
        description: |
          Extra options to be added to LIT_OPTS.
        default: ''

      install_drivers:
        type: choice
        options:
          - false
          - true

      use_dev_igc:
        type: choice
        options:
          - false
          - true

permissions:
  contents: read

jobs:
  run:
    name: ${{ inputs.name }}
    runs-on: ${{ fromJSON(inputs.runner) }}
    container:
      image: ${{ inputs.image }}
      options: ${{ inputs.image_options }}
    env: ${{ fromJSON(inputs.env) }}
    steps:
    - name: Reset GPU
      if: inputs.reset_gpu == 'true'
      run: |
        sudo mount -t debugfs none /sys/kernel/debug
        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
    - uses: actions/checkout@v4
      with:
        ref: ${{ inputs.ref }}
        sparse-checkout: |
          devops
    - name: Register cleanup after job is finished
      uses: ./devops/actions/cleanup
    - name: Checkout E2E tests
      if: inputs.tests_selector == 'e2e'
      uses: ./devops/actions/cached_checkout
      with:
        path: llvm
        ref: ${{ inputs.ref || github.sha }}
        merge_ref: ${{ inputs.merge_ref }}
        cache_path: "/__w/repo_cache/"
    - name: Checkout SYCL CTS tests
      if: inputs.tests_selector == 'cts'
      uses: ./devops/actions/cached_checkout
      with:
        path: khronos_sycl_cts
        repository: 'KhronosGroup/SYCL-CTS'
        ref: 'SYCL-2020'
        default_branch: 'SYCL-2020'
        cache_path: "/__w/repo_cache/"
    - name: SYCL CTS GIT submodules init
      if: inputs.tests_selector == 'cts'
      run: |
        git -C khronos_sycl_cts submodule update --init
    - name: Install drivers
      if: inputs.install_drivers == 'true'
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        if [ "${{ inputs.use_dev_igc }}" = "true" ]; then
            # If libllvm14 is already installed (dev igc docker), still return true.
            sudo apt-get install -yqq libllvm14 || true;
        fi
        sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all
    - name: Source OneAPI TBB vars.sh
      shell: bash
      run: |
        # https://github.com/actions/runner/issues/1964 prevents us from using
        # the ENTRYPOINT in the image.
        env | sort > env_before
        if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
          source /runtimes/oneapi-tbb/env/vars.sh;
        elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
          source /opt/runtimes/oneapi-tbb/env/vars.sh;
        else
          echo "no TBB vars in /opt/runtimes or /runtimes";
        fi
        env | sort > env_after
        comm -13 env_before env_after >> $GITHUB_ENV
        rm env_before env_after
    - name: Download SYCL toolchain
      if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
      uses: actions/download-artifact@v4
      with:
        name: ${{ inputs.sycl_toolchain_artifact }}
    - name: Debug prints [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      run: |
        pwd
        ls
    - name: Download SYCL toolchain [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      uses: actions/github-script@v7
      with:
        script: |
          const name = '${{ inputs.sycl_toolchain_artifact }}'
          let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
             owner: context.repo.owner,
             repo: context.repo.repo,
             run_id: context.payload.workflow_run.id,
          });
          let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
            return artifact.name == name
          })[0];
          let download = await github.rest.actions.downloadArtifact({
             owner: context.repo.owner,
             repo: context.repo.repo,
             artifact_id: matchArtifact.id,
             archive_format: 'zip',
          });
          let fs = require('fs');
          fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data));
    - name: Unzip artifact [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      run: |
        pwd
        ls
        unzip ${{ inputs.sycl_toolchain_artifact }}.zip
        rm ${{ inputs.sycl_toolchain_artifact }}.zip
    - name: Extract/Setup SYCL toolchain
      if: inputs.sycl_toolchain_artifact != ''
      run: |
        mkdir toolchain
        tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain
        rm -f ${{ inputs.sycl_toolchain_archive }}
        echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV
        echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
    - run: which clang++ sycl-ls
    - run: sycl-ls --verbose
    - run: SYCL_PI_TRACE=-1 sycl-ls
    - run: |
          if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then
            cat /usr/local/lib/igc/IGCTAG.txt
          fi

    - name: Deduce E2E CMake options
      if: inputs.tests_selector == 'e2e'
      id: cmake_opts
      shell: bash
      env:
        CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }}
      run: |
        if [ -n "$CMAKE_EXTRA_ARGS" ]; then
          echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT
        else
          if [ "${{ contains(inputs.target_devices, 'ext_oneapi_hip')  }}" == "true" ]; then
            echo 'opts=-DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031"' >> $GITHUB_OUTPUT
          else
            echo 'opts=' >> $GITHUB_OUTPUT
          fi
        fi
    - name: Configure E2E tests
      if: inputs.tests_selector == 'e2e'
      run: |
        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
    - name: SYCL End-to-end tests
      shell: bash {0}
      if: inputs.tests_selector == 'e2e'
      env:
        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
      run: |
        ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
        exit_code=$?
        cat e2e.log
        if [ $exit_code -ne 0 ]; then
          awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY
        fi
        exit $exit_code
    - name: List excluded SYCL CTS categories
      if: inputs.tests_selector == 'cts' && env.CTS_TESTS_TO_BUILD == ''
      run: |
        cat $PWD/devops/cts_exclude_filter
    - name: Build SYCL CTS tests
      if: inputs.tests_selector == 'cts'
      env:
        CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }}
      run: |
        # Do not exclude any category if passing the CTS_TESTS_TO_BUILD variable.
        cts_exclude_filter=$([ -n "$CTS_TESTS_TO_BUILD" ] && echo "" || echo "$PWD/devops/cts_exclude_filter")

        cmake -GNinja -B./build-cts -S./khronos_sycl_cts -DCMAKE_CXX_COMPILER=$(which clang++) \
        -DSYCL_IMPLEMENTATION=DPCPP \
        -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$cts_exclude_filter" \
        -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \
        -DSYCL_CTS_MEASURE_BUILD_TIMES=ON \
        -DDPCPP_INSTALL_DIR="$(dirname $(which clang++))/.." \
        $CMAKE_EXTRA_ARGS
        # Ignore errors so that if one category build fails others still have a
        # chance to finish and be executed at the run stage. Note that
        # "test_conformance" target skips building "test_all" executable.
        ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance")

    - name: SYCL CTS List devices
      # Proceed with execution even if the 'build' step did not succeed.
      if: inputs.tests_selector == 'cts' && (success() || failure())
      env:
        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
      run: |
        ./build-cts/bin/* --list-devices

    - name: Run SYCL CTS tests
      # Proceed with execution even if the previous two steps did not succeed.
      if: inputs.tests_selector == 'cts' && (success() || failure())
      env:
        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
      # This job takes ~100min usually. But sometimes some test isn't
      # responding, so the job reaches the 360min limit. Setting a lower one.
      timeout-minutes: 150
      # By-default GitHub actions execute the "run" shell script with -e option,
      # so the execution terminates if any command returns a non-zero status.
      # Since we're using a loop to run all test-binaries separately, some test
      # may fail and terminate the execution. Setting "shell" value to override
      # the default behavior.
      # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#custom-shell
      shell: bash {0}
      run: |
        # Run each test category separately so that
        #  - crash on one would not affect others
        #  - multiple tests could be run in parallel
        mkdir logs
        find build-cts/bin/ -type f -print | \
          xargs -t -I % -P 8 sh -c 'log=logs/$(basename %).log ; echo % >$log ; date >>$log ; timeout 60m % >>$log 2>&1 ; ret=$? ; echo "exit code: $ret" >>$log ; date >>$log  ; exit $ret'
        ret=$?

        for f in logs/* ; do
          echo "::group::$f"
          cat $f
          echo "::endgroup::"
        done

        echo "::group::Fails:"
        grep 'exit code: [^0]' -r logs
        echo "::endgroup::"

        grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY

        exit $ret

    - name: Run benchmarks (via sycl-bench)
      id: run_benchmarks
      if: inputs.tests_selector == 'benchmark'
      run: ./devops/scripts/sycl-bench.sh https://github.com/ianayl/sycl-bench
    - name: Upload benchmark results
      if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success'
      uses: actions/upload-artifact@v4
      with:
        name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }}
        path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }}
        retention-days: 7