build-hpc.yml

name: build-hpc

# Controls when the action will run
on:

  # Trigger the workflow on all pushes to main and develop, except on tag creation
  push:
    branches:
    - main
    - develop
    tags-ignore:
    - '**'

  # Trigger the workflow on all pull requests
  pull_request: ~

  # Allow workflow to be dispatched on demand
  workflow_dispatch: ~

env:
  ECTRANS_TOOLS: ${{ github.workspace }}/.github/tools
  CTEST_PARALLEL_LEVEL: 1
  CACHE_SUFFIX: v1        # Increase to force new cache to be created

jobs:
  ci-hpc:
    name: ci-hpc

    strategy:
      fail-fast: false    # false: try to complete all jobs

      matrix:
        name:
          - ac-gpu nvhpc
          - lumi-g cce

        include:
          - name: ac-gpu nvhpc
            site: ac-batch
            troika_user_secret: HPC_CI_SSH_USER
            sbatch_options: |
              #SBATCH --time=00:20:00
              #SBATCH --nodes=1
              #SBATCH --ntasks=2
              #SBATCH --cpus-per-task=32
              #SBATCH --gpus-per-task=1
              #SBATCH --mem=200G
              #SBATCH --qos=dg
            modules:
              - cmake
              - ninja
              - prgenv/nvidia
              - hpcx-openmpi/2.14.0-cuda
              - fftw
            workdir: "$TMPDIR"
            output_dir: "$TMPDIR/output"

          - name: lumi-g cce
            site: lumi
            troika_user_secret: LUMI_CI_SSH_USER
            sbatch_options: |
              #SBATCH --time=00:20:00
              #SBATCH --nodes=1
              #SBATCH --ntasks-per-node=8
              #SBATCH --gpus-per-task=1
              #SBATCH --partition=small-g
            modules:
              - CrayEnv
              - PrgEnv-cray
              - cce/17.0.1
              - craype-accel-amd-gfx90a
              - rocm/6.0.3
              - cray-fftw
              - buildtools
            workdir: /tmp
            output_dir: /tmp/output

    runs-on: [self-hosted, linux, hpc]
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: ecmwf-actions/reusable-workflows/ci-hpc-generic@v2
        with:
          site: ${{ matrix.site }}
          troika_user: ${{ secrets[matrix.troika_user_secret] }}
          sbatch_options: ${{ matrix.sbatch_options }}
          output_dir: ${{ matrix.output_dir }}
          workdir: ${{ matrix.workdir }}
          template_data: |
            cmake_options:
              - -DENABLE_MPI=ON
              - -DENABLE_ACC=ON
              - -DENABLE_GPU=ON
            dependencies:
              ecmwf/ecbuild:
                version: develop
              ecmwf-ifs/fiat:
                version: develop
                cmake_options:
                  - -DENABLE_MPI=ON
          template: |
            {% for module in "${{ join(matrix.modules, ',') }}".split(',') %}
              module load {{module}}
            {% endfor %}
            BASEDIR=$PWD
            {% for name, options in dependencies.items() %}
                mkdir -p {{name}}
                pushd {{name}}
                git init
                git remote add origin ${{ github.server_url }}/{{name}}
                git fetch origin {{options['version']}}
                git reset --hard FETCH_HEAD
                cmake -G Ninja -S . -B build \
                  {% for name in dependencies %}
                    {% set org, proj = name.split('/') %}
                    -D{{proj}}_ROOT=$BASEDIR/{{name}}/installation \
                  {% endfor %}
                  {{ options['cmake_options']|join(' ') }}
                cmake --build build
                cmake --install build --prefix installation
                popd
            {% endfor %}
            mkdir -p ${{ github.repository }}
            pushd ${{ github.repository }}
            git init
            git remote add origin ${{ github.server_url }}/${{ github.repository }}
            git fetch origin ${{ github.sha }}
            git reset --hard FETCH_HEAD
            popd
            cmake -G Ninja -S ${{ github.repository }} -B build \
              {% for name in dependencies %}
                {% set org, proj = name.split('/') %}
                -D{{proj}}_ROOT=$BASEDIR/{{name}}/installation \
              {% endfor %}
              {{ cmake_options|join(' ') }}
            cmake --build build
            ctest --test-dir build