CCE 18 on Frontier & Benchmarking fixes among others (#596)

Co-authored-by: Steve Abbott <[email protected]>

.github/workflows/frontier/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 . ./mfc.sh load -c f -m g
-./mfc.sh build -j 8 --gpu
+./mfc.sh build -j 8 --gpu --sys-hdf5 --sys-fftw

.github/workflows/frontier/test.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 
-./mfc.sh test -j 4 -a -- -c frontier
+gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
+ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
+
+./mfc.sh test -j $ngpus --sys-hdf5 --sys-fftw -- -c frontier

.github/workflows/spelling.yml

@@ -11,6 +11,4 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Spell Check
-      uses: crate-ci/typos@master
-      with:
-        config: .typos.toml
+      run:  ./mfc.sh spelling

.gitignore

@@ -3,6 +3,7 @@ package.json
 yarn.lock
 docker-compose.yml
 
+.venv/
 /build/
 .vscode/
 src/*/include/case.fpp
@@ -35,26 +36,27 @@ docs/documentation/*-example.png
 docs/documentation/examples.md
 
 examples/*batch/*/
-examples/*/D/*
-examples/*/p*
-examples/*/D_*
-examples/*/*.inf
-examples/*/*.inp
-examples/*/*.dat
-examples/*/*.o*
-examples/*/silo*
-examples/*/restart_data*
-examples/*/*.out
-examples/*/binary
-examples/*/fort.1
-examples/*/*.sh
-examples/*/*.err
-examples/*/viz/
+examples/**/D/*
+examples/**/p*
+examples/**/D_*
+examples/**/*.inf
+examples/**/*.inp
+examples/**/*.dat
+examples/**/*.o*
+examples/**/silo*
+examples/**/restart_data*
+examples/**/*.out
+examples/**/binary
+examples/**/fort.1
+examples/**/*.sh
+examples/**/*.err
+examples/**/viz/
 examples/*.jpg
 examples/*.png
 examples/*/workloads/
 examples/*/run-*/
 examples/*/logs/
+examples/**/*.f90
 workloads/
 
 benchmarks/*batch/*/

.vscode/settings.json

@@ -7,8 +7,8 @@
         "**/.git":      true,
         "**/.DS_Store": true,
         "**/build":     true,
-        "*.o":          true,
-        "*.mod":        true
+        "**/*.o":       true,
+        "**/*.mod":     true,
     },
 
     "cmake.configureOnOpen": false,
@@ -17,7 +17,7 @@
 
     "files.associations": {
         "*.f90": "FortranFreeForm",
-        "*.fpp": "FortranFreeForm"
+        "*.fpp": "FortranFreeForm",
     },
 
     "fortran.preferredCase":       "lowercase",
@@ -27,5 +27,5 @@
         "${workspacefolder}/src/common",
         "${workspacefolder}/src/common/include"
     ],
-    "fortran.linter.extraArgs":    ["--free-form"]
+    "fortran.linter.extraArgs":    ["--free-form"],
 }

CMakeLists.txt

@@ -55,24 +55,25 @@ compiler (or a different version thereof), please:\n\
  - Install the compiler or load its module. (e.g. module load gcc/10.1)\n\
  - Set/Export the C, CXX, and FC environment variables. (e.g. 'export CC=gcc', \
  'export CXX=g++', and 'export FC=gfortran'.\n\
- - If using mfc.sh, delete the build/<code name> directory and try again. (e.g. 'rm -rf build/pre_process')\n")
+ - If using mfc.sh, delete the build/<code name> directory and try again. (e.g. 'rm -rf build/pre_process')")
 
 if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 5)
-        message(FATAL_ERROR "${__err_msg}ERROR: GNU v5.0 or newer is required to build MFC.")
+        message(FATAL_ERROR "ERROR: GNU v5.0 or newer is required to build MFC.\n${__err_msg}")
     endif()
     if (MFC_OpenACC)
-        message(FATAL_ERROR "${__err_msg}ERROR: MFC with GPU processing is not currently compatible with GNU compilers. Please use NVIDIA or Cray compilers.")
+        message(FATAL_ERROR "ERROR: MFC with GPU processing is not currently compatible with GNU compilers. Please use NVIDIA or Cray compilers.\n${__err_msg}")
     endif()
 elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 21.7)
-        message(FATAL_ERROR "${__err_msg}ERROR: When using NVHPC, v21.7 or newer is required to build MFC.")
+        message(FATAL_ERROR "ERROR: When using NVHPC, v21.7 or newer is required to build MFC.\n${__err_msg}")
     endif()
-    if ((CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
-        message(FATAL_ERROR "${__err_msg}ERROR: When using NVHPC, MFC with Debug and GPU options is unavailable.")
+
+    if ((CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 23.11) AND (CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
+        message(FATAL_ERROR "ERROR: When using NVHPC, MFC with Debug and GPU options requires NVHPC v23.11 or newer.\n${__err_msg}")
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_C_COMPILER_ID STREQUAL "AppleClang")
-    message(FATAL_ERROR "${__err_msg}ERROR: MFC does not support the Apple Clang compilers. Please consult the documentation.")
+    message(FATAL_ERROR "ERROR: MFC does not support the Apple Clang compilers. Please consult the documentation.\n${__err_msg}")
 endif()
 
 # Fypp is required to build MFC. We try to locate it. The path to the binary is
@@ -127,13 +128,15 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
             -fbacktrace
             -fimplicit-none
             #-ffpe-trap=invalid,zero,denormal,overflow
-        )
+            -fsignaling-nans
+	    )
     endif()
 
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER 10)
         add_compile_options(
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-invalid-boz>
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-argument-mismatch>
+            $<$<COMPILE_LANGUAGE:Fortran>:-fcheck=bounds>            
         )
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
@@ -183,7 +186,7 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
     endif()
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-        add_compile_options(-C -g -O0 -traceback -Mchkptr -Minform=inform -Mbounds)
+        add_compile_options(-C -g -O0 -traceback -Minform=inform -Mbounds)
     endif()
 
     if (DEFINED ENV{MFC_CUDA_CC})

examples/3D_sphbubcollapse/case.py

@@ -45,7 +45,7 @@
     # ==========================================================================
 
     # Simulation Algorithm Parameters ==========================================
-    # Only two patches are necesssary, the background liquid and the
+    # Only two patches are necessary, the background liquid and the
     # gas bubble
     'num_patches'                  : 2,
     # Use the 6 equation model

mfc.sh

@@ -38,6 +38,10 @@ elif [ "$1" '==' "venv" ]; then
     shift; . "$(pwd)/toolchain/bootstrap/python.sh"  $@; return
 elif [ "$1" '==' "clean" ]; then
     rm -rf "$(pwd)/build"; exit 0
+elif [ "$1" '==' "spelling" ]; then
+    . "$(pwd)/toolchain/bootstrap/python.sh"
+
+    shift; . "$(pwd)/toolchain/bootstrap/spelling.sh" $@; exit 0
 fi
 
 mkdir -p "$(pwd)/build"

src/pre_process/m_data_output.fpp

@@ -367,6 +367,12 @@ contains
             end if
         end if
 
+        if (precision == 1) then
+            FMT = "(4F30.7)"
+        else
+            FMT = "(4F40.14)"
+        end if
+
         ! 3D
         if (p > 0) then
             do i = 1, sys_size

src/simulation/m_fftw.fpp

@@ -57,9 +57,9 @@ module m_fftw
     @:CRAY_DECLARE_GLOBAL(complex(kind(0d0)), dimension(:), data_fltr_cmplx_gpu)
     !$acc declare link(data_real_gpu, data_cmplx_gpu, data_fltr_cmplx_gpu)
 #else
-    real(kind(0d0)), allocatable :: data_real_gpu(:)
-    complex(kind(0d0)), allocatable :: data_cmplx_gpu(:)
-    complex(kind(0d0)), allocatable :: data_fltr_cmplx_gpu(:)
+    real(kind(0d0)), allocatable, target :: data_real_gpu(:)
+    complex(kind(0d0)), allocatable, target :: data_cmplx_gpu(:)
+    complex(kind(0d0)), allocatable, target :: data_fltr_cmplx_gpu(:)
     !$acc declare create(data_real_gpu, data_cmplx_gpu, data_fltr_cmplx_gpu)
 #endif
 
@@ -141,7 +141,8 @@ contains
     subroutine s_apply_fourier_filter(q_cons_vf)
 
         type(scalar_field), dimension(sys_size), intent(inout) :: q_cons_vf
-
+        real(c_double), pointer :: p_real(:)
+        complex(c_double_complex), pointer :: p_cmplx(:), p_fltr_cmplx(:)
         integer :: i, j, k, l !< Generic loop iterators
 
         ! Restrict filter to processors that have cells adjacent to axis
@@ -166,11 +167,16 @@ contains
             end do
         end do
 
-!$acc host_data use_device(data_real_gpu, data_cmplx_gpu)
+        p_real => data_real_gpu
+        p_cmplx => data_cmplx_gpu
+        p_fltr_cmplx => data_fltr_cmplx_gpu
+
+!$acc data attach(p_real, p_cmplx, p_fltr_cmplx)
+!$acc host_data use_device(p_real, p_cmplx, p_fltr_cmplx)
 #if defined(__PGI)
         ierr = cufftExecD2Z(fwd_plan_gpu, data_real_gpu, data_cmplx_gpu)
 #else
-        ierr = hipfftExecD2Z(fwd_plan_gpu, c_loc(data_real_gpu), c_loc(data_cmplx_gpu))
+        ierr = hipfftExecD2Z(fwd_plan_gpu, c_loc(p_real), c_loc(p_cmplx))
         call hipCheck(hipDeviceSynchronize())
 #endif
         !$acc end host_data
@@ -186,11 +192,11 @@ contains
             end do
         end do
 
-!$acc host_data use_device(data_real_gpu, data_fltr_cmplx_gpu)
+!$acc host_data use_device(p_real, p_fltr_cmplx)
 #if defined(__PGI)
         ierr = cufftExecZ2D(bwd_plan_gpu, data_fltr_cmplx_gpu, data_real_gpu)
 #else
-        ierr = hipfftExecZ2D(bwd_plan_gpu, c_loc(data_fltr_cmplx_gpu), c_loc(data_real_gpu))
+        ierr = hipfftExecZ2D(bwd_plan_gpu, c_loc(p_fltr_cmplx), c_loc(p_real))
         call hipCheck(hipDeviceSynchronize())
 #endif
         !$acc end host_data
@@ -225,11 +231,11 @@ contains
                 end do
             end do
 
-!$acc host_data use_device(data_real_gpu, data_cmplx_gpu)
+!$acc host_data use_device(p_real, p_cmplx)
 #if defined(__PGI)
             ierr = cufftExecD2Z(fwd_plan_gpu, data_real_gpu, data_cmplx_gpu)
 #else
-            ierr = hipfftExecD2Z(fwd_plan_gpu, c_loc(data_real_gpu), c_loc(data_cmplx_gpu))
+            ierr = hipfftExecD2Z(fwd_plan_gpu, c_loc(p_real), c_loc(p_cmplx))
             call hipCheck(hipDeviceSynchronize())
 #endif
             !$acc end host_data
@@ -246,11 +252,11 @@ contains
                 end do
             end do
 
-!$acc host_data use_device(data_real_gpu, data_fltr_cmplx_gpu)
+!$acc host_data use_device(p_real, p_fltr_cmplx)
 #if defined(__PGI)
             ierr = cufftExecZ2D(bwd_plan_gpu, data_fltr_cmplx_gpu, data_real_gpu)
 #else
-            ierr = hipfftExecZ2D(bwd_plan_gpu, c_loc(data_fltr_cmplx_gpu), c_loc(data_real_gpu))
+            ierr = hipfftExecZ2D(bwd_plan_gpu, c_loc(p_fltr_cmplx), c_loc(p_real))
             call hipCheck(hipDeviceSynchronize())
 #endif
             !$acc end host_data
@@ -297,8 +303,8 @@ contains
             end do
         end do
 #endif
-
-    end subroutine s_apply_fourier_filter
+!$acc end data
+    end subroutine s_apply_fourier_filter ! --------------------------------
 
     !>  The purpose of this subroutine is to destroy the fftw plan
         !!      that will be used in the forward and backward DFTs when