Merge branch 'cusini/scec-benchmarks' of github.com:GEOS-DEV/GEOS int…

…o cusini/scec-benchmarks
GEOS-DEV · Jan 31, 2025 · d6bd871 · d6bd871
2 parents 3af1996 + 3fdd2a7
commit d6bd871
Show file tree

Hide file tree

Showing 35 changed files with 441 additions and 347 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,7 +2,7 @@
     "build": {
         "dockerfile": "Dockerfile",
         "args": {
-            "GEOS_TPL_TAG": "288-584"
+            "GEOS_TPL_TAG": "290-594"
         }
     },
     "runArgs": [

diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
@@ -183,6 +183,15 @@ jobs:
             ENABLE_TRILINOS: OFF
             BUILD_SHARED_LIBS: ON
 
+          - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10)
+            CMAKE_BUILD_TYPE: Release
+            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
+            ENABLE_HYPRE: ON
+            ENABLE_TRILINOS: OFF
+            GCP_BUCKET: geosx/Sherlock-CPU
+            HOST_CONFIG: host-configs/Stanford/sherlock-gcc10.cmake
+            BUILD_SHARED_LIBS: ON
+
     uses: ./.github/workflows/build_and_test.yml
     with:
       BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }}
@@ -192,6 +201,7 @@ jobs:
       ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }}
       ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }}
       GCP_BUCKET: ${{ matrix.GCP_BUCKET }}
+      HOST_CONFIG: ${{ matrix.HOST_CONFIG }}
       RUNS_ON: ubuntu-22.04
     secrets: inherit
 
@@ -346,6 +356,21 @@ jobs:
             DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
             DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"
 
+          - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, cuda 12.4.0,)
+            BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema"
+            BUILD_GENERATOR: "--ninja"
+            CMAKE_BUILD_TYPE: Release
+            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11
+            ENABLE_HYPRE_DEVICE: CUDA
+            ENABLE_HYPRE: ON
+            ENABLE_TRILINOS: OFF
+            GCP_BUCKET: geosx/Sherlock-GPU
+            HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-cuda12-sm70.cmake
+            RUNS_ON: streak2
+            NPROC: 8
+            DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro"
+            DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
+            DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"
 
           # Below this line, jobs that deploy to Google Cloud.
 

diff --git a/host-configs/Stanford/sherlock-base.cmake b/host-configs/Stanford/sherlock-base.cmake
diff --git a/host-configs/Stanford/sherlock-gcc10-cuda11-ampere.cmake b/host-configs/Stanford/sherlock-gcc10-cuda11-ampere.cmake
diff --git a/host-configs/Stanford/sherlock-gcc10-cuda11-volta.cmake b/host-configs/Stanford/sherlock-gcc10-cuda11-volta.cmake
diff --git a/....1.2-openblas0.3.10-cuda11.7.1-sm70.cmake → ...Stanford/sherlock-gcc10-cuda12-sm70.cmake b/....1.2-openblas0.3.10-cuda11.7.1-sm70.cmake → ...Stanford/sherlock-gcc10-cuda12-sm70.cmake
@@ -1,17 +1,14 @@
-include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake)
-
-# OpenMP options
-set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10.cmake)
 
 # CUDA options
 set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(CUDA_VERSION "11.7.1" CACHE PATH "")
+set(CUDA_VERSION "12.4.0" CACHE PATH "")
 set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "")
 set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
 set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "")
 set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "")
 
-set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
+set(CONFIG_NAME "sherlock-gcc${GCC_VERSION}-ompi${OMPI_VERSION}-openblas${OPENBLAS_VERSION}-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
 
 set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
 set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
@@ -20,6 +17,9 @@ set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" C
 set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
 
+# OpenMP options
+set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+
 # LAI options
 set(GEOS_LA_INTERFACE "Hypre" CACHE STRING "" FORCE)
 set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)

diff --git a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake b/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake
diff --git a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake b/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
diff --git a/host-configs/Stanford/sherlock-gcc10.cmake b/host-configs/Stanford/sherlock-gcc10.cmake
@@ -1,7 +1,45 @@
-set(CONFIG_NAME "sherlock-gcc10" CACHE PATH "") 
+# User defined versions
+set(GCC_VERSION "10.1.0" CACHE STRING "Version of GCC")
+set(OMPI_VERSION "4.1.2" CACHE STRING "Version of Open MPI")
+set(OPENBLAS_VERSION "0.3.10" CACHE STRING "Version of OpenBLAS")
 
-set(GCC_ROOT "/share/software/user/open/gcc/10.1.0" CACHE PATH "")
-set(MPI_ROOT "/share/software/user/open/openmpi/4.1.0" CACHE PATH "")
+set(CONFIG_NAME "sherlock-gcc${GCC_VERSION}-ompi${OMPI_VERSION}-openblas${OPENBLAS_VERSION}" CACHE PATH "")
 
-include(/home/groups/tchelepi/geosx/host-configs/sherlock-base.cmake)
+set(SOFTWARE_ROOT /share/software/user/open CACHE PATH "")
+set(GCC_ROOT "${SOFTWARE_ROOT}/gcc/${GCC_VERSION}" CACHE PATH "")
+set(MPI_ROOT "${SOFTWARE_ROOT}/openmpi/${OMPI_VERSION}" CACHE PATH "")
 
+site_name(HOST_NAME)
+
+# Compilers
+set(CMAKE_C_COMPILER       "${GCC_ROOT}/bin/gcc"      CACHE PATH "")
+set(CMAKE_CXX_COMPILER     "${GCC_ROOT}/bin/g++"      CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/gfortran" CACHE PATH "")
+
+# OpenMP options
+#set(ENABLE_OPENMP ON CACHE BOOL "")
+
+# MPI options
+set(ENABLE_MPI ON CACHE PATH "" FORCE)
+set(MPI_C_COMPILER       "${MPI_ROOT}/bin/mpicc"   CACHE PATH "")
+set(MPI_CXX_COMPILER     "${MPI_ROOT}/bin/mpic++"  CACHE PATH "")
+set(MPI_Fortran_COMPILER "${MPI_ROOT}/bin/mpifort" CACHE PATH "")
+set(MPIEXEC              "${MPI_ROOT}/bin/mpirun"  CACHE PATH "")
+set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
+set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "")
+
+# CUDA options
+set(ENABLE_CUDA OFF CACHE PATH "" FORCE)
+
+# Blas/Lapack options
+set(OPENBLAS_ROOT "${SOFTWARE_ROOT}/openblas/${OPENBLAS_VERSION}" CACHE STRING "")
+set(BLAS_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
+set(LAPACK_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
+
+set(ENABLE_VALGRIND OFF CACHE BOOL "")
+set(ENABLE_CALIPER ON CACHE BOOL "")
+
+if (DEFINED ENV{GEOSX_TPL_DIR})
+    set(GEOS_TPL_DIR "$ENV{GEOSX_TPL_DIR}" CACHE PATH "" FORCE)
+endif()
+include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)
diff --git a/host-configs/Stanford/sherlock-gcc8-cuda10-volta.cmake b/host-configs/Stanford/sherlock-gcc8-cuda10-volta.cmake
diff --git a/host-configs/Stanford/sherlock-modules.sh b/host-configs/Stanford/sherlock-modules.sh
@@ -1,3 +1,6 @@
+module --force purge 
+module load devel
+module load math
 module load system
 module load git
 module load git-lfs
@@ -9,4 +12,6 @@ module load libevent/2.1.12 # needed for silo
 module load py-mpi4py/3.1.3_py39
 module load py-h5py/3.7.0_py39
 module unload cuda
-module load cuda/11.7.1
+module load cuda/12.4.0 
+module load ninja/1.9.0
+
diff --git a/host-configs/apple/macOS_base.cmake b/host-configs/apple/macOS_base.cmake
@@ -25,7 +25,7 @@ set(ENABLE_CALIPER "OFF" CACHE PATH "" FORCE )
 set( BLAS_LIBRARIES ${HOMEBREW_DIR}/opt/lapack/lib/libblas.dylib CACHE PATH "" FORCE )
 set( LAPACK_LIBRARIES ${HOMEBREW_DIR}/opt/lapack/lib/liblapack.dylib CACHE PATH "" FORCE )
 
-set(ENABLE_DOXYGEN ON CACHE BOOL "" FORCE)
+set(ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE)
 set(ENABLE_SPHINX ON CACHE BOOL "" FORCE)
 set(ENABLE_MATHPRESSO ON CACHE BOOL "" FORCE )