gpu-ci.yml

name: "GPU-based Tests"
on:
  push:
    branches:
      - "inference"
  workflow_dispatch:

concurrency:
  group: gpu-ci-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  python-interface-check:
    name: Check Python Interface
    runs-on: "runs-on=${{ github.run_id }}/runner=gpu-nvidia"
    defaults:
      run:
        shell: bash -l {0} # required to use an activated conda environment
    env: 
      CONDA: "3"
    # container:
    #   image: ghcr.io/flexflow/flexflow-environment-cuda-12.2:latest
    #   options: --gpus all --shm-size=8192m
    steps:
      - name: Display NVIDIA SMI and NVCC details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
          nvcc --version
      
      - name: Display disk space
        run: df -h
      
      - name: Install updated git version
        run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt update -y && sudo apt install -y --no-install-recommends git

      - name: Checkout Git Repository
        uses: actions/checkout@v4
        with:
          submodules: recursive
          
      - name: Install conda and FlexFlow dependencies
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: "latest"
          activate-environment: flexflow
          environment-file: conda/flexflow.yml
          auto-activate-base: false
          auto-update-conda: false

      - name: Build FlexFlow
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          mkdir build
          cd build
          ../config/config.linux
          make -j

      - name: Check FlexFlow Python interface (before installation)
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib
          ./tests/python_interface_test.sh before-installation

      - name: Install FlexFlow
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          cd build
          ../config/config.linux
          make install
          ldconfig

      - name: Check FlexFlow Python interface (after installation)
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib
          ./tests/python_interface_test.sh after-installation

  inference-tests:
    name: Inference Tests
    runs-on: "runs-on=${{ github.run_id }}/runner=gpu-nvidia"
    defaults:
      run:
        shell: bash -l {0} # required to use an activated conda environment
    env: 
      CONDA: "3"
      HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
    # container:
    #   image: ghcr.io/flexflow/flexflow-environment-cuda-12.2:latest
    #   options: --gpus all --shm-size=8192m
    steps:
      - name: Display NVIDIA SMI and NVCC details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
          nvcc --version
      
      - name: Display disk space
        run: df -h
      
      - name: Install updated git version
        run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt update -y && sudo apt install -y --no-install-recommends git

      - name: Checkout Git Repository
        uses: actions/checkout@v4
        with:
          submodules: recursive
          
      - name: Install conda and FlexFlow dependencies
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: "latest"
          activate-environment: flexflow
          environment-file: conda/flexflow.yml
          auto-activate-base: false
          auto-update-conda: false

      - name: Build FlexFlow
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          mkdir build
          cd build
          ../config/config.linux
          make -j
      
      - name: Run inference tests
        env:
          CPP_INFERENCE_TESTS: ${{ vars.CPP_INFERENCE_TESTS }}
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib

          # Inference tests
          source ./build/set_python_envs.sh
          ./tests/inference_tests.sh
      
      - name: Run PEFT tests
        run: |
          export PATH=$CONDA_PREFIX/bin:$PATH
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib

          source ./build/set_python_envs.sh
          ./tests/peft_test.sh
      
      - name: Save inference output as an artifact
        if: always()
        run: |
          cd inference
          tar -zcvf output.tar.gz ./output

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: output
          path: inference/output.tar.gz
      
      # Github persists the .cache folder across different runs/containers
      - name: Clear cache
        if: always()
        run: sudo rm -rf ~/.cache