test_iree.yml

# Copyright 2024 Advanced Micro Devices, Inc.
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: IREE Test Suite
on:
  # TODO(scotttodd): run on schedule (nightly), and/or on pushes to `main`
  pull_request:
    paths:
      # This file itself.
      - ".github/workflows/test_iree.yml"
      # The iree_tests subproject.
      - "iree_tests/**"
  workflow_dispatch:

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit). The workflow name is prepended to avoid conflicts between
  # different workflows.
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true


jobs:
  onnx:
    name: "ONNX :: ${{ matrix.name }} :: ${{ matrix.runs-on }}"
    runs-on: ${{ matrix.runs-on }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: cpu_llvm_sync
            runs-on: ubuntu-24.04
            config-file: config_onnx_cpu_llvm_sync.json
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
      CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.config-file }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@v4
        with:
          submodules: false
          lfs: true

      # Install Python packages.
      - name: "Setting up Python"
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: "Setup Python venv"
        run: python3 -m venv ${VENV_DIR}
      - name: "Installing IREE nightly release Python packages"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade \
            iree-compiler \
            iree-runtime
      - name: "Installing Python test requirements"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install -r iree_tests/requirements.txt

      # Run tests and output new config files as needed.
      - name: "Running the generated ONNX test suite"
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/onnx/ \
            -n auto \
            -rpfE \
            --timeout=30 \
            --durations=10 \
            --report-log=/tmp/iree_tests_onnx_${{ matrix.name }}_logs.json \
            --config-files=${CONFIG_FILE_PATH}
      - name: "Updating config file with latest XFAIL lists"
        if: failure()
        run: |
          source ${VENV_DIR}/bin/activate
          python iree_tests/update_config_xfails.py \
            --log-file=/tmp/iree_tests_onnx_${{ matrix.name }}_logs.json \
            --config-file=${CONFIG_FILE_PATH}
          cat ${CONFIG_FILE_PATH}
      - name: "Uploading new config file"
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.config-file }}
          path: ${{ env.CONFIG_FILE_PATH }}

  models:
    name: "Models :: ${{ matrix.name }} :: ${{ matrix.runs-on }}"
    runs-on: ${{ matrix.runs-on }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: cpu_llvm_task
            runs-on: nodai-amdgpu-w7900-x86-64
            models-config-file: config_pytorch_models_cpu_llvm_task.json
            sdxl-prompt-encoder-config-file: config_sd_prompt_encoder_cpu_llvm_task.json
            sdxl-unet-config-file: config_sd_scheduled_unet_cpu_llvm_task.json
            sdxl-vae-decode-config-file: config_sd_vae_decode_cpu_llvm_task.json
            backend: cpu
          - name: gpu_mi250_rocm
            runs-on: nodai-amdgpu-mi250-x86-64
            models-config-file: config_gpu_rocm_models.json
            sdxl-prompt-encoder-config-file: config_sd_prompt_encoder_gpu_rocm.json
            sdxl-unet-config-file: config_sd_scheduled_unet_gpu_rocm.json
            sdxl-vae-decode-config-file: config_sd_vae_decode_gpu_rocm.json
            backend: rocm
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
      IREE_TEST_FILES: ~/iree_tests_cache
      IREE_TEST_PATH_EXTENSION: ${{ github.workspace }}/iree_tests/specs
      MODELS_CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.models-config-file }}
      SDXL_PROMPT_ENCODER_CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.sdxl-prompt-encoder-config-file }}
      SDXL_UNET_CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.sdxl-unet-config-file }}
      SDXL_VAE_DECODE_CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.sdxl-vae-decode-config-file }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@v4
        with:
          submodules: false
          lfs: true

      # Install Python packages.
      # Note: skipping actions/setup-python since these self-hosted runners already have Python.
      - name: "Setup Python venv"
        run: python3 -m venv ${VENV_DIR}
      - name: "Installing IREE nightly release Python packages"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade \
            iree-compiler \
            iree-runtime
      - name: "Installing Python test requirements"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install -r iree_tests/requirements.txt

      # Download remote files.
      - name: "Downloading remote files for real weight model tests"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 iree_tests/download_remote_files.py --root-dir pytorch/models

      - name: "Running real weights model tests"
        if: ${{ !cancelled() }}
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/pytorch/models \
            -n 4 \
            -rpfE \
            -k real_weights \
            --no-skip-tests-missing-files \
            --capture=no \
            --log-cli-level=info \
            --timeout=1200 \
            --durations=0 \
            --config-files=${MODELS_CONFIG_FILE_PATH}

      - name: "Running real weights SDXL + SD3 prompt encoder tests"
        id: prompt_encoder
        if: ${{ !cancelled() }}
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/pytorch/models/sd-clip \
            -rpfE \
            -k real_weights \
            --no-skip-tests-missing-files \
            --capture=no \
            --log-cli-level=info \
            --timeout=1200 \
            --durations=0 \
            --config-files=${SDXL_PROMPT_ENCODER_CONFIG_FILE_PATH}

      - name: "Running real weights SDXL + SD3 scheduled unet/mmdit tests"
        id: unet
        if: ${{ !cancelled() }}
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/pytorch/models/sd-unet \
            -rpfE \
            -k real_weights \
            --no-skip-tests-missing-files \
            --capture=no \
            --log-cli-level=info \
            --timeout=1200 \
            --durations=0 \
            --config-files=${SDXL_UNET_CONFIG_FILE_PATH}
        env:
          IREE_TEST_BACKEND: ${{ matrix.backend }}

      - name: "Running real weights SDXL + SD3 vae decode tests"
        id: vae
        if: ${{ !cancelled() }}
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/pytorch/models/sd-vae \
            -rpfE \
            -k real_weights \
            --no-skip-tests-missing-files \
            --capture=no \
            --log-cli-level=info \
            --timeout=1200 \
            --durations=0 \
            --config-files=${SDXL_VAE_DECODE_CONFIG_FILE_PATH}

      - name: "Running SDXL ROCm pipeline benchmark"
        if: |
          contains(matrix.name, 'rocm') &&
          steps.prompt_encoder.outcome == 'success' &&
          steps.unet.outcome == 'success' && steps.vae.outcome == 'success'
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/benchmarks/benchmark_sdxl_rocm.py \
            --goldentime-rocm-e2e-ms 1661.5 \
            --goldentime-rocm-unet-ms 450.5 \
            --goldentime-rocm-clip-ms 19 \
            --goldentime-rocm-vae-ms 288.5 \
            --gpu-number 6 \
            --rocm-chip gfx90a \
            --log-cli-level=info \
            --retries 3
          echo "### SDXL Benchmark Summary:" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line
          echo "$(<job_summary.txt )" >> $GITHUB_STEP_SUMMARY