Merge pull request #355 from sfiligoi/tests_240301

Add CGYRO CI

.github/support_scripts/install_hpc_sdk.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+# This is a helper script for installing the NVIDIA HPC SDK 
+# Note: The script currently assumes Linux_x86_64 platform.
+#
+
+# Install the NVIDIA HPC SDK
+
+# This link may need to be updated, as new compiler versions are released
+# Note: Verified that it works with v24.1
+if [ "x${NV_URL}" == "x" ]; then
+  NV_URL=https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_multi.tar.gz
+fi
+
+echo "Downloading the NVIDIA HPC SDK"
+curl -s "${NV_URL}" | tar xpzf -
+
+echo "Installing NVIDIA HPC SDK"
+
+export NVHPC_INSTALL_DIR=$PWD/hpc_sdk
+export NVHPC_SILENT=true
+
+(cd nvhpc_*; ./install)
+
+# create helper scripts
+mkdir setup_scripts
+cat > setup_scripts/setup_nv_hpc_bins.sh << EOF
+module add ./hpc_sdk/modulefiles/nvhpc-openmpi3/*
+
+EOF
+
+echo "Setup script avaiabile in $PWD/setup_scripts/setup_nv_hpc_bins.sh"

.github/workflows/cgyro.yml

@@ -0,0 +1,119 @@
+name: cgyro CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  build-and-test:
+    strategy:
+      matrix:
+        os: [linux-gpu-cuda]
+        hwflavor: [cpu, openacc, ompgpu]
+    runs-on: ${{ matrix.os }}
+    env:
+      hwflavor: ${{ matrix.hwflavor }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniconda-version: "latest"
+        auto-update-conda: true
+    - name: Install
+      shell: bash -l {0}
+      run: |
+        df -h .
+        echo "HW flavor: '${hwflavor}'"
+        if [[ "${hwflavor}" == "cpu" ]]; then
+          conda create -q --yes --strict-channel-priority -n gacode -c conda-forge gxx_linux-64 gfortran_linux-64 make curl python
+          conda clean --yes -t
+          df -h .
+          conda activate gacode
+          which x86_64-conda-linux-gnu-gcc
+          x86_64-conda-linux-gnu-gcc -v
+          x86_64-conda-linux-gnu-g++ -v
+          conda install --yes --strict-channel-priority -c conda-forge fftw 'openblas=*=*openmp*'
+          conda install --yes --strict-channel-priority -c conda-forge mpich
+          export GACODE_PLATFORM=CI_CPU
+        else
+          # Assuming gfortran and modules are pre-installed
+          # install NVIDIA HPC SDK
+          .github/support_scripts/install_hpc_sdk.sh
+          source setup_scripts/setup_nv_hpc_bins.sh
+          export GACODE_PLATFORM=CI_GPU
+          if [[ "${hwflavor}" == "ompgpu" ]]; then
+             export GACODE_OMPGPU=1
+          else
+             export GACODE_OMPGPU=0
+          fi
+        fi
+        df -h .
+        export GACODE_ROOT=$PWD
+        . $GACODE_ROOT/shared/bin/gacode_setup
+        source $GACODE_ROOT/platform/env/env.${GACODE_PLATFORM}
+        echo "======= begin env ====="
+        env
+        echo "=======  end env  ====="
+        # build the code
+        echo "=======  building  ====="
+        (cd cgyro && make)
+        echo "=======  cgyro bin  ====="
+        (cd cgyro/src; ls -l cgyro)
+        (cd cgyro/src; ldd cgyro)
+            
+    - name: Tests
+      shell: bash -l {0}
+      run: |
+        echo "HW flavor: '${hwflavor}'"
+        if [[ "${hwflavor}" == "cpu" ]]; then
+          conda activate gacode
+          export GACODE_PLATFORM=CI_CPU
+        else
+          source setup_scripts/setup_nv_hpc_bins.sh
+          export GACODE_PLATFORM=CI_GPU
+          if [[ "${hwflavor}" == "ompgpu" ]]; then
+             export GACODE_OMPGPU=1
+          else
+             export GACODE_OMPGPU=0
+          fi
+        fi
+        export GACODE_ROOT=$PWD
+        . $GACODE_ROOT/shared/bin/gacode_setup
+        source $GACODE_ROOT/platform/env/env.${GACODE_PLATFORM}
+        echo "======= begin env ====="
+        env
+        echo "=======  end env  ====="
+        echo "=======  cgyro bin  ====="
+        (cd cgyro/src; ls -l cgyro)
+        (cd cgyro/src; ldd cgyro)
+        echo "=======  testing  ====="
+        # test the code
+        (cd cgyro/bin && cgyro -r -n 4 -nomp 2) | tee cgyro_reg.log
+        echo "=======  last out  ====="
+        if [ -f cgyro/bin/cgyro_regression_test/out ]; then cat cgyro/bin/cgyro_regression_test/out; fi
+        echo "=======  results  ====="
+        (grep reg cgyro_reg.log || test -f cgyro_reg.log) |tee  cgyro_reg.tests.log
+        (grep PASS cgyro_reg.tests.log || test -f cgyro_reg.tests.log) > cgyro_reg.tests.pass.log
+        cnt=`wc -l cgyro_reg.tests.pass.log |awk '{print $1}'`
+        if [ "$cnt" -eq 20 ]; then
+          echo "== ALL passed =="
+        else 
+          echo "== Some tests FAILED"
+          find cgyro/bin/cgyro_regression_test
+          echo "==== cgyro_regression_test/out"
+          if [ -f cgyro/bin/cgyro_regression_test/out ]; then cat cgyro/bin/cgyro_regression_test/out; fi
+          echo "==== cgyro_regression_test/.../out.cgyro.info"
+          cat cgyro/bin/cgyro_regression_test/*/out.cgyro.info
+          echo "== Aborting"
+          test -z "error"
+        fi
+
+    - name: Sanity checks
+      shell: bash -l {0}
+      run: |
+        echo "TODO"

cgyro/install/make.ext.CI_GPU

@@ -0,0 +1,2 @@
+cgyro_nl_fftw.o : cgyro_nl_fftw.gpu.F90
+	$(FC) $(FMATH) $(FFLAGS) -o cgyro_nl_fftw.o -c cgyro_nl_fftw.gpu.F90

platform/build/make.inc.CI_GPU

@@ -0,0 +1,33 @@
+IDENTITY="Generic NVHPC GPU"
+CORES_PER_NODE=16
+NUMAS_PER_NODE=1
+
+# Fortran 90/95 compiler
+FC = mpif90 -module ${GACODE_ROOT}/modules -Mpreprocess -DUSE_INLINE -Mdefaultunit
+
+# Fortran 77 compiler
+F77 = ${FC}
+
+# Compiler options/flags
+ifneq ($(GACODE_OMPGPU),1)
+FACC   =-acc -Minfo=accel -cudalib=cufft
+else
+FACC    =-mp=gpu -DOMPGPU -Minfo=mp,accel -cudalib=cufft
+endif
+FOMP   =-mp -Mstack_arrays 
+FMATH  =-r8 
+FOPT   =-fast
+FDEBUG =-g -Kieee -Ktrap=fp,divz -Mbounds -Mchkptr -Mchkstk -traceback -Minform=inform
+F2PY   = f2py --fcompiler=pg
+
+
+# System math libraries
+LMATH=-llapack -lblas
+
+# NetCDF
+NETCDF=-L${NETCDF_DIR}/lib -lnetcdff -lnetcdf
+NETCDF_INC = ${NETCDF_DIR}/include
+
+# Archive 
+ARCH = ar cr
+

platform/env/env.CONDA_CPU → platform/env/env.CI_CPU

@@ -5,8 +5,8 @@ if [ -n "$SSH_TTY" ] ; then
 fi
 
 # Recommended installation options
-# conda create -q --yes --strict-channel-priority -n gacode -c conda-forge gxx_linux-64 gfortran_linux-64 make curl
+# conda create -q --yes --strict-channel-priority -n gacode -c conda-forge gxx_linux-64 gfortran_linux-64 make curl python
 # conda activate gacode
-# conda install --strict-channel-priority -c conda-forge fftw openblas
+# conda install --strict-channel-priority -c conda-forge fftw 'openblas=*=*openmp*'
 # conda install --strict-channel-priority -c conda-forge mpich
 

platform/env/env.CI_GPU

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -n "$SSH_TTY" ] ; then
+   echo "Setting up $GACODE_PLATFORM environment for gacode"
+fi
+
+# Expects NVIDIA HPC SDK  to be installed
+

platform/exec/exec.CI_CPU

@@ -0,0 +1,13 @@
+#!/bin/sh
+# GACODE Parallel execution script
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+
+cd $simdir
+mpiexec -env OMP_NUM_THREADS $nomp -n $nmpi $exec 
+

platform/exec/exec.CI_GPU

@@ -0,0 +1,14 @@
+#!/bin/sh
+# GACODE Parallel execution script
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+
+cd $simdir
+export OMP_NUM_THREADS=$nomp
+mpiexec -oversubscribe -n $nmpi $exec 
+