diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 28ba75bfd2b..193c5fb7a70 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -2,7 +2,7 @@
"build": {
"dockerfile": "Dockerfile",
"args": {
- "GEOS_TPL_TAG": "288-584"
+ "GEOS_TPL_TAG": "290-594"
}
},
"runArgs": [
diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
index 32d77b0cc5e..7a28f313d01 100644
--- a/.github/workflows/ci_tests.yml
+++ b/.github/workflows/ci_tests.yml
@@ -183,6 +183,15 @@ jobs:
ENABLE_TRILINOS: OFF
BUILD_SHARED_LIBS: ON
+ - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10)
+ CMAKE_BUILD_TYPE: Release
+ DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
+ ENABLE_HYPRE: ON
+ ENABLE_TRILINOS: OFF
+ GCP_BUCKET: geosx/Sherlock-CPU
+ HOST_CONFIG: host-configs/Stanford/sherlock-gcc10.cmake
+ BUILD_SHARED_LIBS: ON
+
uses: ./.github/workflows/build_and_test.yml
with:
BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }}
@@ -192,6 +201,7 @@ jobs:
ENABLE_HYPRE: ${{ matrix.ENABLE_HYPRE }}
ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }}
GCP_BUCKET: ${{ matrix.GCP_BUCKET }}
+ HOST_CONFIG: ${{ matrix.HOST_CONFIG }}
RUNS_ON: ubuntu-22.04
secrets: inherit
@@ -346,6 +356,21 @@ jobs:
DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"
+ - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, cuda 12.4.0,)
+ BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema"
+ BUILD_GENERATOR: "--ninja"
+ CMAKE_BUILD_TYPE: Release
+ DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11
+ ENABLE_HYPRE_DEVICE: CUDA
+ ENABLE_HYPRE: ON
+ ENABLE_TRILINOS: OFF
+ GCP_BUCKET: geosx/Sherlock-GPU
+ HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-cuda12-sm70.cmake
+ RUNS_ON: streak2
+ NPROC: 8
+ DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro"
+ DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
+ DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"
# Below this line, jobs that deploy to Google Cloud.
diff --git a/host-configs/Stanford/sherlock-base.cmake b/host-configs/Stanford/sherlock-base.cmake
deleted file mode 100644
index e7e0e0ee83a..00000000000
--- a/host-configs/Stanford/sherlock-base.cmake
+++ /dev/null
@@ -1,50 +0,0 @@
-site_name(HOST_NAME)
-
-# Compilers
-set(CMAKE_C_COMPILER "${GCC_ROOT}/bin/gcc" CACHE PATH "")
-set(CMAKE_CXX_COMPILER "${GCC_ROOT}/bin/g++" CACHE PATH "")
-set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/gfortran" CACHE PATH "")
-
-# OpenMP options
-set(ENABLE_OPENMP ON CACHE BOOL "")
-
-# MPI options
-set(ENABLE_MPI ON CACHE PATH "" FORCE)
-set(MPI_C_COMPILER "${MPI_ROOT}/bin/mpicc" CACHE PATH "")
-set(MPI_CXX_COMPILER "${MPI_ROOT}/bin/mpic++" CACHE PATH "")
-set(MPI_Fortran_COMPILER "${MPI_ROOT}/bin/mpifort" CACHE PATH "")
-set(MPIEXEC "${MPI_ROOT}/bin/mpirun" CACHE PATH "")
-set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
-set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "")
-
-# CUDA options
-if(ENABLE_CUDA)
- set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
- set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
- set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
- set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "")
- set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
- set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
-endif()
-
-# Blas/Lapack options
-set(BLAS_LIBRARIES "/share/software/user/open/openblas/0.3.10/lib/libblas.so" CACHE STRING "")
-set(LAPACK_LIBRARIES "/share/software/user/open/openblas/0.3.10/lib/liblapack.so" CACHE STRING "")
-
-# Python options
-#set(ENABLE_PYLVARRAY ON CACHE BOOL "")
-#set(ENABLE_PYGEOSX ON CACHE BOOL "")
-#set(PYTHON_DIR "/share/software/user/open/python/3.6.1" CACHE PATH "")
-#set(Python3_EXECUTABLE "/share/software/user/open/python/3.6.1/bin/python3" CACHE PATH "")
-
-set(ENABLE_VALGRIND OFF CACHE BOOL "")
-set(ENABLE_CALIPER ON CACHE BOOL "")
-
-if( ${ENABLE_HYPRE_DEVICE} STREQUAL "CUDA" )
- set(ENABLE_PETSC OFF CACHE BOOL "")
- set(ENABLE_TRILINOS OFF CACHE BOOL "")
- set(GEOS_LA_INTERFACE "Hypre" CACHE STRING "")
-endif()
-
-set(GEOS_TPL_DIR /home/groups/tchelepi/geosx/thirdPartyLibs/install-${CONFIG_NAME}-release CACHE PATH "")
-include(/home/groups/tchelepi/geosx/GEOSX/host-configs/tpls.cmake)
diff --git a/host-configs/Stanford/sherlock-gcc10-cuda11-ampere.cmake b/host-configs/Stanford/sherlock-gcc10-cuda11-ampere.cmake
deleted file mode 100644
index 62889988335..00000000000
--- a/host-configs/Stanford/sherlock-gcc10-cuda11-ampere.cmake
+++ /dev/null
@@ -1,13 +0,0 @@
-set(CONFIG_NAME "sherlock-gcc10-cuda11-ampere" CACHE PATH "")
-
-set(GCC_ROOT "/share/software/user/open/gcc/10.1.0" CACHE PATH "")
-set(MPI_ROOT "/share/software/user/open/openmpi/4.1.0" CACHE PATH "")
-
-set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE)
-set(CUDA_ARCH "sm_80" CACHE STRING "")
-set(CMAKE_CUDA_ARCHITECTURES "80" CACHE STRING "")
-set(CUDA_TOOLKIT_ROOT_DIR "/share/software/user/open/cuda/11.2.0" CACHE STRING "")
-
-include(/home/groups/tchelepi/geosx/host-configs/sherlock-base.cmake)
-
diff --git a/host-configs/Stanford/sherlock-gcc10-cuda11-volta.cmake b/host-configs/Stanford/sherlock-gcc10-cuda11-volta.cmake
deleted file mode 100644
index 1de72e32fe5..00000000000
--- a/host-configs/Stanford/sherlock-gcc10-cuda11-volta.cmake
+++ /dev/null
@@ -1,13 +0,0 @@
-set(CONFIG_NAME "sherlock-gcc10-cuda11-volta" CACHE PATH "")
-
-set(GCC_ROOT "/share/software/user/open/gcc/10.1.0" CACHE PATH "")
-set(MPI_ROOT "/share/software/user/open/openmpi/4.1.0" CACHE PATH "")
-
-set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE)
-set(CUDA_ARCH "sm_70" CACHE STRING "")
-set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
-set(CUDA_TOOLKIT_ROOT_DIR "/share/software/user/open/cuda/11.2.0" CACHE STRING "")
-
-include(/home/groups/tchelepi/geosx/host-configs/sherlock-base.cmake)
-
diff --git a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake b/host-configs/Stanford/sherlock-gcc10-cuda12-sm70.cmake
similarity index 83%
rename from host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake
rename to host-configs/Stanford/sherlock-gcc10-cuda12-sm70.cmake
index 0fd52e8d3bf..153a177e37c 100644
--- a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake
+++ b/host-configs/Stanford/sherlock-gcc10-cuda12-sm70.cmake
@@ -1,17 +1,14 @@
-include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake)
-
-# OpenMP options
-set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10.cmake)
# CUDA options
set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(CUDA_VERSION "11.7.1" CACHE PATH "")
+set(CUDA_VERSION "12.4.0" CACHE PATH "")
set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "")
set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "")
set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "")
-set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
+set(CONFIG_NAME "sherlock-gcc${GCC_VERSION}-ompi${OMPI_VERSION}-openblas${OPENBLAS_VERSION}-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
@@ -20,6 +17,9 @@ set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" C
set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
+# OpenMP options
+set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+
# LAI options
set(GEOS_LA_INTERFACE "Hypre" CACHE STRING "" FORCE)
set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
diff --git a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake b/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake
deleted file mode 100644
index 4d33559f0ad..00000000000
--- a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake
+++ /dev/null
@@ -1,28 +0,0 @@
-include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake)
-
-# OpenMP options
-set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
-
-# CUDA options
-set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(CUDA_VERSION "11.5.0" CACHE PATH "")
-set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "")
-set(CMAKE_CUDA_ARCHITECTURES "80" CACHE STRING "")
-set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "")
-set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "")
-
-set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
-
-set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
-set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
-set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr -Werror cross-execution-space-call,reorder,deprecated-declarations " CACHE STRING "")
-set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "")
-set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
-set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
-
-# LAI options
-set(GEOS_LA_INTERFACE "Hypre" CACHE STRING "" FORCE)
-set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE)
-set(ENABLE_PETSC OFF CACHE BOOL "" FORCE)
-set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE)
diff --git a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake b/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
deleted file mode 100644
index d29b6180e11..00000000000
--- a/host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
+++ /dev/null
@@ -1,43 +0,0 @@
-set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10" CACHE PATH "")
-
-set(SOFTWARE_ROOT /share/software/user/open CACHE PATH "")
-set(GCC_ROOT "${SOFTWARE_ROOT}/gcc/10.1.0" CACHE PATH "")
-set(MPI_ROOT "${SOFTWARE_ROOT}/openmpi/4.1.2" CACHE PATH "")
-
-site_name(HOST_NAME)
-
-# Compilers
-set(CMAKE_C_COMPILER "${GCC_ROOT}/bin/gcc" CACHE PATH "")
-set(CMAKE_CXX_COMPILER "${GCC_ROOT}/bin/g++" CACHE PATH "")
-set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/gfortran" CACHE PATH "")
-
-# OpenMP options
-set(ENABLE_OPENMP ON CACHE BOOL "")
-
-# MPI options
-set(ENABLE_MPI ON CACHE PATH "" FORCE)
-set(MPI_C_COMPILER "${MPI_ROOT}/bin/mpicc" CACHE PATH "")
-set(MPI_CXX_COMPILER "${MPI_ROOT}/bin/mpic++" CACHE PATH "")
-set(MPI_Fortran_COMPILER "${MPI_ROOT}/bin/mpifort" CACHE PATH "")
-set(MPIEXEC "${MPI_ROOT}/bin/mpirun" CACHE PATH "")
-set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
-set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "")
-
-# CUDA options
-set(ENABLE_CUDA OFF CACHE PATH "" FORCE)
-
-# Blas/Lapack options
-set(OPENBLAS_ROOT "${SOFTWARE_ROOT}/openblas/0.3.10" CACHE STRING "")
-set(BLAS_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
-set(LAPACK_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
-
-# Python options
-#set(ENABLE_PYLVARRAY ON CACHE BOOL "")
-#set(ENABLE_PYGEOSX ON CACHE BOOL "")
-#set(PYTHON_DIR "${SOFTWARE_ROOT}/python/3.6.1" CACHE PATH "")
-#set(Python3_EXECUTABLE "${SOFTWARE_ROOT}/python/3.6.1/bin/python3" CACHE PATH "")
-
-set(ENABLE_VALGRIND OFF CACHE BOOL "")
-set(ENABLE_CALIPER ON CACHE BOOL "")
-
-include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)
diff --git a/host-configs/Stanford/sherlock-gcc10.cmake b/host-configs/Stanford/sherlock-gcc10.cmake
index aa6308538b7..b1590980f04 100644
--- a/host-configs/Stanford/sherlock-gcc10.cmake
+++ b/host-configs/Stanford/sherlock-gcc10.cmake
@@ -1,7 +1,45 @@
-set(CONFIG_NAME "sherlock-gcc10" CACHE PATH "")
+# User defined versions
+set(GCC_VERSION "10.1.0" CACHE STRING "Version of GCC")
+set(OMPI_VERSION "4.1.2" CACHE STRING "Version of Open MPI")
+set(OPENBLAS_VERSION "0.3.10" CACHE STRING "Version of OpenBLAS")
-set(GCC_ROOT "/share/software/user/open/gcc/10.1.0" CACHE PATH "")
-set(MPI_ROOT "/share/software/user/open/openmpi/4.1.0" CACHE PATH "")
+set(CONFIG_NAME "sherlock-gcc${GCC_VERSION}-ompi${OMPI_VERSION}-openblas${OPENBLAS_VERSION}" CACHE PATH "")
-include(/home/groups/tchelepi/geosx/host-configs/sherlock-base.cmake)
+set(SOFTWARE_ROOT /share/software/user/open CACHE PATH "")
+set(GCC_ROOT "${SOFTWARE_ROOT}/gcc/${GCC_VERSION}" CACHE PATH "")
+set(MPI_ROOT "${SOFTWARE_ROOT}/openmpi/${OMPI_VERSION}" CACHE PATH "")
+site_name(HOST_NAME)
+
+# Compilers
+set(CMAKE_C_COMPILER "${GCC_ROOT}/bin/gcc" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "${GCC_ROOT}/bin/g++" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/gfortran" CACHE PATH "")
+
+# OpenMP options
+#set(ENABLE_OPENMP ON CACHE BOOL "")
+
+# MPI options
+set(ENABLE_MPI ON CACHE PATH "" FORCE)
+set(MPI_C_COMPILER "${MPI_ROOT}/bin/mpicc" CACHE PATH "")
+set(MPI_CXX_COMPILER "${MPI_ROOT}/bin/mpic++" CACHE PATH "")
+set(MPI_Fortran_COMPILER "${MPI_ROOT}/bin/mpifort" CACHE PATH "")
+set(MPIEXEC "${MPI_ROOT}/bin/mpirun" CACHE PATH "")
+set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
+set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "")
+
+# CUDA options
+set(ENABLE_CUDA OFF CACHE PATH "" FORCE)
+
+# Blas/Lapack options
+set(OPENBLAS_ROOT "${SOFTWARE_ROOT}/openblas/${OPENBLAS_VERSION}" CACHE STRING "")
+set(BLAS_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
+set(LAPACK_LIBRARIES "${OPENBLAS_ROOT}/lib/libopenblas.so" CACHE STRING "")
+
+set(ENABLE_VALGRIND OFF CACHE BOOL "")
+set(ENABLE_CALIPER ON CACHE BOOL "")
+
+if (DEFINED ENV{GEOSX_TPL_DIR})
+ set(GEOS_TPL_DIR "$ENV{GEOSX_TPL_DIR}" CACHE PATH "" FORCE)
+endif()
+include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)
diff --git a/host-configs/Stanford/sherlock-gcc8-cuda10-volta.cmake b/host-configs/Stanford/sherlock-gcc8-cuda10-volta.cmake
deleted file mode 100644
index 18213ea9857..00000000000
--- a/host-configs/Stanford/sherlock-gcc8-cuda10-volta.cmake
+++ /dev/null
@@ -1,13 +0,0 @@
-set(CONFIG_NAME "sherlock-gcc8-cuda10-volta" CACHE PATH "")
-
-set(GCC_ROOT "/share/software/user/open/gcc/8.1.0" CACHE PATH "")
-set(MPI_ROOT "/share/software/user/open/openmpi/4.0.3" CACHE PATH "")
-
-set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_DEVICE CUDA CACHE STRING "" FORCE)
-set(CUDA_ARCH "sm_70" CACHE STRING "")
-set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
-set(CUDA_TOOLKIT_ROOT_DIR "/share/software/user/open/cuda/10.2.89" CACHE STRING "")
-
-include(/home/groups/tchelepi/geosx/host-configs/sherlock-base.cmake)
-
diff --git a/host-configs/Stanford/sherlock-modules.sh b/host-configs/Stanford/sherlock-modules.sh
index 316116271e2..17a5453d8d0 100755
--- a/host-configs/Stanford/sherlock-modules.sh
+++ b/host-configs/Stanford/sherlock-modules.sh
@@ -1,3 +1,6 @@
+module --force purge
+module load devel
+module load math
module load system
module load git
module load git-lfs
@@ -9,4 +12,6 @@ module load libevent/2.1.12 # needed for silo
module load py-mpi4py/3.1.3_py39
module load py-h5py/3.7.0_py39
module unload cuda
-module load cuda/11.7.1
+module load cuda/12.4.0
+module load ninja/1.9.0
+
diff --git a/host-configs/apple/macOS_base.cmake b/host-configs/apple/macOS_base.cmake
index ec74d15b9c1..6d56f70755c 100644
--- a/host-configs/apple/macOS_base.cmake
+++ b/host-configs/apple/macOS_base.cmake
@@ -25,7 +25,7 @@ set(ENABLE_CALIPER "OFF" CACHE PATH "" FORCE )
set( BLAS_LIBRARIES ${HOMEBREW_DIR}/opt/lapack/lib/libblas.dylib CACHE PATH "" FORCE )
set( LAPACK_LIBRARIES ${HOMEBREW_DIR}/opt/lapack/lib/liblapack.dylib CACHE PATH "" FORCE )
-set(ENABLE_DOXYGEN ON CACHE BOOL "" FORCE)
+set(ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE)
set(ENABLE_SPHINX ON CACHE BOOL "" FORCE)
set(ENABLE_MATHPRESSO ON CACHE BOOL "" FORCE )
diff --git a/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_base.xml b/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_base.xml
index 1257f1ae3ec..12e05d26fe6 100644
--- a/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_base.xml
+++ b/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_base.xml
@@ -18,9 +18,9 @@
name="mesh1"
elementTypes="{ C3D8 }"
xCoords="{ -10.0e3, 10.0e3 }"
- yCoords="{ -0.5e3, 0.5e3 }"
- zCoords="{ -20.0e3, -10.0e3, 10.0e3, 20.0e3 }"
- nx="{ 100 }"
+ yCoords="{ -0.1e3, 0.1e3 }"
+ zCoords="{ -40.0e3, -10.0e3, 10.0e3, 40.0e3 }"
+ nx="{ 50 }"
ny="{ 1 }"
nz="{ 20, 201, 20 }"
cellBlockNames="{ cb1 }"/>
@@ -29,13 +29,58 @@
+ xMin="{ -0.01, -1.01e3, -20.0e3 }"
+ xMax="{ 0.01, 1.01e3, 20.0e3 }"/>
+
+
+
+ xMin="{ -0.01, -1.01e3, -0.1e3 }"
+ xMax="{ 0.01, 1.01e3, 0.1e3 }"/>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -47,7 +92,7 @@
+ defaultAperture="5e-3"/>
@@ -83,7 +128,7 @@
name="rock"
defaultDensity="2670"
defaultBulkModulus="50.0e9"
- defaultShearModulus="32.04e9"/>
+ defaultShearModulus="24.03e9"/>
+ setNames="{ receiver1, receiver2, receiver3, receiver4, receiver5, receiver6, receiver7, source}"/>
+ setNames="{ receiver1, receiver2, receiver3, receiver4, receiver5, receiver6, receiver7, source}"/>
@@ -169,7 +214,7 @@
objectPath="ElementRegions/Fault"
fieldName="stateVariable"
scale="0.68070857"
- setNames="{ all }"
+ setNames="{ faultPlane }"
initialCondition="1"/>
+ setNames="{ faultPlane }"/>
+ setNames="{ faultPlane }"/>
+ setNames="{ faultPlane }"/>
+ setNames="{ faultPlane }"/>
\ No newline at end of file
diff --git a/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_explicit.xml b/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_explicit.xml
index ffab615c0f9..025017d8bff 100644
--- a/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_explicit.xml
+++ b/inputFiles/inducedSeismicity/SCEC_BP6_QD_S_explicit.xml
@@ -56,8 +56,8 @@
lineSearchMaxCuts="2"
maxTimeStepCuts="1"/>
@@ -77,14 +77,16 @@
newtonTol = "1.0e-6"
newtonMaxIter ="8"/>
+ solverType="gmres"
+ preconditionerType="amg"
+ logLevel="0"/>
-
+
+ maxTime="1e6">
-
-
+ maxTime="1e6">
@@ -128,6 +129,8 @@ struct MpiWrapper
Min, //!< Min
Sum, //!< Sum
Prod, //!< Prod
+ LogicalAnd, //!< Logical and
+ LogicalOr, //!< Logical or
};
MpiWrapper() = delete;
@@ -351,18 +354,6 @@ struct MpiWrapper
array1d< T > & recvbuf,
MPI_Comm comm = MPI_COMM_GEOS );
- /**
- * @brief Strongly typed wrapper around MPI_Allreduce.
- * @param[in] sendbuf The pointer to the sending buffer.
- * @param[out] recvbuf The pointer to the receive buffer.
- * @param[in] count The number of values to send/receive.
- * @param[in] op The MPI_Op to perform.
- * @param[in] comm The MPI_Comm over which the gather operates.
- * @return The return value of the underlying call to MPI_Allreduce().
- */
- template< typename T >
- static int allReduce( T const * sendbuf, T * recvbuf, int count, MPI_Op op, MPI_Comm comm = MPI_COMM_GEOS );
-
/**
* @brief Convenience wrapper for the MPI_Allreduce function.
* @tparam T type of data to reduce. Must correspond to a valid MPI_Datatype.
@@ -385,6 +376,29 @@ struct MpiWrapper
template< typename T >
static void allReduce( Span< T const > src, Span< T > dst, Reduction const op, MPI_Comm comm = MPI_COMM_GEOS );
+ /**
+ * @brief Convenience wrapper for the MPI_Allreduce function. Version for arrays.
+ * @tparam T type of data to reduce. Must correspond to a valid MPI_Datatype.
+ * @param src[in] The values to send to the reduction.
+ * @param dst[out] The resulting values.
+ * @param op The Reduction enum to perform.
+ * @param comm The communicator.
+ */
+ template< typename SRC_CONTAINER_TYPE, typename DST_CONTAINER_TYPE >
+ static void allReduce( SRC_CONTAINER_TYPE const & src, DST_CONTAINER_TYPE & dst, Reduction const op, MPI_Comm const comm = MPI_COMM_GEOS );
+
+ /**
+ * @brief Convenience wrapper for the MPI_Allreduce function. Version for arrays.
+ * @tparam T type of data to reduce. Must correspond to a valid MPI_Datatype.
+ * @param src[in] The values to send to the reduction.
+ * @param dst[out] The resulting values.
+ * @param count The number of contiguos elements of the arrays to perform the reduction on (must be leq than the size).
+ * @param op The Reduction enum to perform.
+ * @param comm The communicator.
+ */
+ template< typename SRC_CONTAINER_TYPE, typename DST_CONTAINER_TYPE >
+ static void allReduce( SRC_CONTAINER_TYPE const & src, DST_CONTAINER_TYPE & dst, int const count, Reduction const op, MPI_Comm const comm );
+
/**
* @brief Strongly typed wrapper around MPI_Reduce.
@@ -639,6 +653,19 @@ struct MpiWrapper
*/
template< typename T > static T maxValLoc( T localValueLocation, MPI_Comm comm = MPI_COMM_GEOS );
+private:
+
+ /**
+ * @brief Strongly typed wrapper around MPI_Allreduce.
+ * @param[in] sendbuf The pointer to the sending buffer.
+ * @param[out] recvbuf The pointer to the receive buffer.
+ * @param[in] count The number of values to send/receive.
+ * @param[in] op The MPI_Op to perform.
+ * @param[in] comm The MPI_Comm over which the gather operates.
+ * @return The return value of the underlying call to MPI_Allreduce().
+ */
+ template< typename T >
+ static int allReduce( T const * sendbuf, T * recvbuf, int count, MPI_Op op, MPI_Comm comm = MPI_COMM_GEOS );
};
namespace internal
@@ -701,6 +728,14 @@ inline MPI_Op MpiWrapper::getMpiOp( Reduction const op )
{
return MPI_PROD;
}
+ case Reduction::LogicalAnd:
+ {
+ return MPI_LAND;
+ }
+ case Reduction::LogicalOr:
+ {
+ return MPI_LOR;
+ }
default:
GEOS_ERROR( "Unsupported reduction operation" );
return MPI_NO_OP;
@@ -1113,6 +1148,35 @@ void MpiWrapper::allReduce( Span< T const > const src, Span< T > const dst, Redu
allReduce( src.data(), dst.data(), LvArray::integerConversion< int >( src.size() ), getMpiOp( op ), comm );
}
+template< typename SRC_CONTAINER_TYPE, typename DST_CONTAINER_TYPE >
+void MpiWrapper::allReduce( SRC_CONTAINER_TYPE const & src, DST_CONTAINER_TYPE & dst, int const count, Reduction const op, MPI_Comm const comm )
+{
+ static_assert( std::is_trivially_copyable< typename get_value_type< SRC_CONTAINER_TYPE >::type >::value,
+ "The type in the source container must be trivially copyable." );
+ static_assert( std::is_trivially_copyable< typename get_value_type< DST_CONTAINER_TYPE >::type >::value,
+ "The type in the destination container must be trivially copyable." );
+ static_assert( std::is_same< typename get_value_type< SRC_CONTAINER_TYPE >::type,
+ typename get_value_type< DST_CONTAINER_TYPE >::type >::value,
+ "Source and destination containers must have the same value type." );
+ GEOS_ASSERT_GE( src.size(), count );
+ GEOS_ASSERT_GE( dst.size(), count );
+ allReduce( src.data(), dst.data(), count, getMpiOp( op ), comm );
+}
+
+template< typename SRC_CONTAINER_TYPE, typename DST_CONTAINER_TYPE >
+void MpiWrapper::allReduce( SRC_CONTAINER_TYPE const & src, DST_CONTAINER_TYPE & dst, Reduction const op, MPI_Comm const comm )
+{
+ static_assert( std::is_trivially_copyable< typename get_value_type< SRC_CONTAINER_TYPE >::type >::value,
+ "The type in the source container must be trivially copyable." );
+ static_assert( std::is_trivially_copyable< typename get_value_type< DST_CONTAINER_TYPE >::type >::value,
+ "The type in the destination container must be trivially copyable." );
+ static_assert( std::is_same< typename get_value_type< SRC_CONTAINER_TYPE >::type,
+ typename get_value_type< DST_CONTAINER_TYPE >::type >::value,
+ "Source and destination containers must have the same value type." );
+ GEOS_ASSERT_EQ( src.size(), dst.size() );
+ allReduce( src.data(), dst.data(), LvArray::integerConversion< int >( src.size() ), getMpiOp( op ), comm );
+}
+
template< typename T >
T MpiWrapper::sum( T const & value, MPI_Comm comm )
{
diff --git a/src/coreComponents/common/TypesHelpers.hpp b/src/coreComponents/common/TypesHelpers.hpp
new file mode 100644
index 00000000000..f0e514e35a1
--- /dev/null
+++ b/src/coreComponents/common/TypesHelpers.hpp
@@ -0,0 +1,124 @@
+/*
+ * ------------------------------------------------------------------------------------------------------------
+ * SPDX-License-Identifier: LGPL-2.1-only
+ *
+ * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC
+ * Copyright (c) TotalEnergies
+ * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University
+ * Copyright (c) 2023-2024 Chevron
+ * Copyright (c) 2019- GEOS/GEOSX Contributors
+ * All rights reserved
+ *
+ * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details.
+ * ------------------------------------------------------------------------------------------------------------
+ */
+
+/**
+ * @file TypesHelpers.hpp
+ *
+ */
+
+#ifndef TYPES_HELPERS_HPP
+#define TYPES_HELPERS_HPP
+
+#include
+
+namespace geos
+{
+
+namespace internal
+{
+/**
+ * @brief Trait to determine if a type defines a `value_type` member.
+ *
+ * This primary template defaults to `std::false_type`, indicating that
+ * the type `T` does not define a `value_type` member.
+ *
+ * @tparam T The type to check.
+ * @tparam void A SFINAE parameter used to specialize the trait.
+ */
+template< typename T, typename = void >
+struct has_value_type : std::false_type {};
+
+/**
+ * @brief Specialization of `has_value_type` for types with a `value_type` member.
+ *
+ * If the type `T` defines a `value_type` member, this specialization
+ * is used, which inherits from `std::true_type`.
+ *
+ * @tparam T The type to check.
+ */
+template< typename T >
+struct has_value_type< T, std::void_t< typename T::value_type > > : std::true_type {};
+
+/**
+ * @brief Trait to determine if a type defines a `ValueType` member.
+ *
+ * This primary template defaults to `std::false_type`, indicating that
+ * the type `T` does not define a `ValueType` member.
+ *
+ * @tparam T The type to check.
+ * @tparam void A SFINAE parameter used to specialize the trait.
+ */
+template< typename T, typename = void >
+struct has_ValueType : std::false_type {};
+
+/**
+ * @brief Specialization of `has_ValueType` for types with a `ValueType` member.
+ *
+ * If the type `T` defines a `ValueType` member, this specialization
+ * is used, which inherits from `std::true_type`.
+ *
+ * @tparam T The type to check.
+ */
+template< typename T >
+struct has_ValueType< T, std::void_t< typename T::ValueType > > : std::true_type {};
+
+} // namespace internal
+
+/**
+ * @brief Trait to retrieve the `value_type` or `ValueType` of a type `T`.
+ *
+ * This primary template provides a static assertion error if `T` does not
+ * define either `value_type` or `ValueType`.
+ *
+ * @tparam T The type from which to extract the type alias.
+ * @tparam Enable A SFINAE parameter used for specialization.
+ */
+template< typename T, typename Enable = void >
+struct get_value_type
+{
+ static_assert( sizeof(T) == 0, "T must define either value_type or ValueType." );
+};
+
+/**
+ * @brief Specialization of `get_value_type` for types with a `value_type` member.
+ *
+ * If the type `T` defines a `value_type` member, this specialization
+ * retrieves it as the alias `type`.
+ *
+ * @tparam T The type from which to extract `value_type`.
+ */
+template< typename T >
+struct get_value_type< T, std::enable_if_t< internal::has_value_type< T >::value > >
+{
+ using type = typename T::value_type;
+};
+
+/**
+ * @brief Specialization of `get_value_type` for types with a `ValueType` member.
+ *
+ * If the type `T` does not define a `value_type` but defines a `ValueType`,
+ * this specialization retrieves it as the alias `type`.
+ *
+ * @tparam T The type from which to extract `ValueType`.
+ */
+template< typename T >
+struct get_value_type< T, std::enable_if_t< !internal::has_value_type< T >::value && internal::has_ValueType< T >::value > >
+{
+ using type = typename T::ValueType;
+};
+
+} // namespace geos
+
+#endif /* TYPES_HELPERS_HPP */
diff --git a/src/coreComponents/common/initializeEnvironment.cpp b/src/coreComponents/common/initializeEnvironment.cpp
index 832f4fa542f..64387060583 100644
--- a/src/coreComponents/common/initializeEnvironment.cpp
+++ b/src/coreComponents/common/initializeEnvironment.cpp
@@ -290,7 +290,7 @@ static void addUmpireHighWaterMarks()
string allocatorNameMinChars = string( MAX_NAME_LENGTH, '\0' );
// Make sure that each rank is looking at the same allocator.
- MpiWrapper::allReduce( allocatorNameFixedSize.c_str(), &allocatorNameMinChars.front(), MAX_NAME_LENGTH, MPI_MIN, MPI_COMM_GEOS );
+ MpiWrapper::allReduce( allocatorNameFixedSize, allocatorNameMinChars, MpiWrapper::Reduction::Min, MPI_COMM_GEOS );
if( allocatorNameFixedSize != allocatorNameMinChars )
{
GEOS_WARNING( "Not all ranks have an allocator named " << allocatorNameFixedSize << ", cannot compute high water mark." );
diff --git a/src/coreComponents/fileIO/timeHistory/HDFHistoryIO.cpp b/src/coreComponents/fileIO/timeHistory/HDFHistoryIO.cpp
index 24ae0ddddf6..fdfbf95c61d 100644
--- a/src/coreComponents/fileIO/timeHistory/HDFHistoryIO.cpp
+++ b/src/coreComponents/fileIO/timeHistory/HDFHistoryIO.cpp
@@ -258,8 +258,9 @@ void HDFHistoryIO::init( bool existsOkay )
void HDFHistoryIO::write()
{
// check if the size has changed on any process in the primary comm
- int anyChanged = false;
- MpiWrapper::allReduce( &m_sizeChanged, &anyChanged, 1, MPI_LOR, m_comm );
+ int const anyChanged = MpiWrapper::allReduce( m_sizeChanged,
+ MpiWrapper::Reduction::LogicalOr,
+ m_comm );
m_sizeChanged = anyChanged;
// this will set the first dim large enough to hold all the rows we're about to write
diff --git a/src/coreComponents/finiteVolume/TwoPointFluxApproximation.cpp b/src/coreComponents/finiteVolume/TwoPointFluxApproximation.cpp
index 25c128770fa..665e1eed140 100644
--- a/src/coreComponents/finiteVolume/TwoPointFluxApproximation.cpp
+++ b/src/coreComponents/finiteVolume/TwoPointFluxApproximation.cpp
@@ -951,10 +951,9 @@ void TwoPointFluxApproximation::computeAquiferStencil( DomainPartition & domain,
localSumFaceAreasView[aquiferIndex] += targetSetSumFaceAreas.get();
} );
- MpiWrapper::allReduce( localSumFaceAreas.data(),
- globalSumFaceAreas.data(),
- localSumFaceAreas.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+ MpiWrapper::allReduce( localSumFaceAreas,
+ globalSumFaceAreas,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
// Step 3: compute the face area fraction for each connection, and insert into boundary stencil
diff --git a/src/coreComponents/mesh/ParticleManager.cpp b/src/coreComponents/mesh/ParticleManager.cpp
index 057533e4f7e..57bd1687935 100644
--- a/src/coreComponents/mesh/ParticleManager.cpp
+++ b/src/coreComponents/mesh/ParticleManager.cpp
@@ -77,11 +77,9 @@ void ParticleManager::setMaxGlobalIndex()
m_localMaxGlobalIndex = std::max( m_localMaxGlobalIndex, subRegion.maxGlobalIndex() );
} );
- MpiWrapper::allReduce( &m_localMaxGlobalIndex,
- &m_maxGlobalIndex,
- 1,
- MPI_MAX,
- MPI_COMM_GEOS );
+ m_maxGlobalIndex = MpiWrapper::allReduce( m_localMaxGlobalIndex,
+ MpiWrapper::Reduction::Max,
+ MPI_COMM_GEOS );
}
Group * ParticleManager::createChild( string const & childKey, string const & childName )
diff --git a/src/coreComponents/mesh/generators/InternalMeshGenerator.cpp b/src/coreComponents/mesh/generators/InternalMeshGenerator.cpp
index ace6dd8846e..99fe38fb059 100644
--- a/src/coreComponents/mesh/generators/InternalMeshGenerator.cpp
+++ b/src/coreComponents/mesh/generators/InternalMeshGenerator.cpp
@@ -603,10 +603,9 @@ void InternalMeshGenerator::fillCellBlockManager( CellBlockManager & cellBlockMa
{
elemCenterCoordsLocal[k] = m_min[dim] + ( m_max[dim] - m_min[dim] ) * ( k + 0.5 ) / m_numElemsTotal[dim];
}
- MpiWrapper::allReduce( elemCenterCoordsLocal.data(),
- elemCenterCoords[dim].data(),
- m_numElemsTotal[dim],
- MPI_MAX,
+ MpiWrapper::allReduce( elemCenterCoordsLocal,
+ elemCenterCoords[dim],
+ MpiWrapper::Reduction::Max,
MPI_COMM_GEOS );
}
diff --git a/src/coreComponents/mesh/generators/VTKUtilities.cpp b/src/coreComponents/mesh/generators/VTKUtilities.cpp
index a18b3e6767a..1fb198d70ce 100644
--- a/src/coreComponents/mesh/generators/VTKUtilities.cpp
+++ b/src/coreComponents/mesh/generators/VTKUtilities.cpp
@@ -746,8 +746,7 @@ vtkSmartPointer< vtkDataSet > manageGlobalIds( vtkSmartPointer< vtkDataSet > mes
{
// Add global ids on the fly if needed
int const me = hasGlobalIds( mesh );
- int everyone;
- MpiWrapper::allReduce( &me, &everyone, 1, MPI_MAX, MPI_COMM_GEOS );
+ int const everyone = MpiWrapper::allReduce( me, MpiWrapper::Reduction::Max, MPI_COMM_GEOS );
if( everyone and not me )
{
diff --git a/src/coreComponents/physicsSolvers/PhysicsSolverBaseKernels.hpp b/src/coreComponents/physicsSolvers/PhysicsSolverBaseKernels.hpp
index ac8433b208b..45e2ade8570 100644
--- a/src/coreComponents/physicsSolvers/PhysicsSolverBaseKernels.hpp
+++ b/src/coreComponents/physicsSolvers/PhysicsSolverBaseKernels.hpp
@@ -267,10 +267,9 @@ class LinfResidualNormHelper
static void computeGlobalNorm( array1d< real64 > const & localResidualNorm,
array1d< real64 > & globalResidualNorm )
{
- MpiWrapper::allReduce( localResidualNorm.data(),
- globalResidualNorm.data(),
- localResidualNorm.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Max ),
+ MpiWrapper::allReduce( localResidualNorm,
+ globalResidualNorm,
+ MpiWrapper::Reduction::Max,
MPI_COMM_GEOS );
}
};
@@ -309,16 +308,17 @@ class L2ResidualNormHelper
{
array1d< real64 > sumLocalResidualNorm( localResidualNorm.size() );
array1d< real64 > sumLocalResidualNormalizer( localResidualNormalizer.size() );
- MpiWrapper::allReduce( localResidualNorm.data(),
- sumLocalResidualNorm.data(),
- localResidualNorm.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+
+ MpiWrapper::allReduce( localResidualNorm,
+ sumLocalResidualNorm,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
- MpiWrapper::allReduce( localResidualNormalizer.data(),
- sumLocalResidualNormalizer.data(),
- localResidualNormalizer.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+
+ MpiWrapper::allReduce( localResidualNormalizer,
+ sumLocalResidualNormalizer,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
+
for( integer i = 0; i < localResidualNorm.size(); ++i )
{
globalResidualNorm[i] = sqrt( sumLocalResidualNorm[i] ) / sqrt( sumLocalResidualNormalizer[i] );
diff --git a/src/coreComponents/physicsSolvers/contact/ContactSolverBase.cpp b/src/coreComponents/physicsSolvers/contact/ContactSolverBase.cpp
index edd3bd4ec24..9555658db65 100644
--- a/src/coreComponents/physicsSolvers/contact/ContactSolverBase.cpp
+++ b/src/coreComponents/physicsSolvers/contact/ContactSolverBase.cpp
@@ -184,10 +184,9 @@ void ContactSolverBase::computeFractureStateStatistics( MeshLevel const & mesh,
array1d< globalIndex > totalCounter( 4 );
- MpiWrapper::allReduce( localCounter.data(),
- totalCounter.data(),
- 4,
- MPI_SUM,
+ MpiWrapper::allReduce( localCounter,
+ totalCounter,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
numStick = totalCounter[0];
diff --git a/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp b/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp
index b78afce691a..4e31c7c4eaa 100644
--- a/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp
+++ b/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp
@@ -841,12 +841,9 @@ bool SolidMechanicsEmbeddedFractures::updateConfiguration( DomainPartition & dom
synchronizeFractureState( domain );
// Compute if globally the fracture state has changed
- int hasConfigurationConvergedGlobally;
- MpiWrapper::allReduce( &hasConfigurationConverged,
- &hasConfigurationConvergedGlobally,
- 1,
- MPI_LAND,
- MPI_COMM_GEOS );
+ int const hasConfigurationConvergedGlobally = MpiWrapper::allReduce( hasConfigurationConverged,
+ MpiWrapper::Reduction::LogicalAnd,
+ MPI_COMM_GEOS );
// for this solver it makes sense to reset the state.
// if( !hasConfigurationConvergedGlobally )
diff --git a/src/coreComponents/physicsSolvers/fluidFlow/FlowSolverBase.cpp b/src/coreComponents/physicsSolvers/fluidFlow/FlowSolverBase.cpp
index e9a55b86c46..35c1a9a6b40 100644
--- a/src/coreComponents/physicsSolvers/fluidFlow/FlowSolverBase.cpp
+++ b/src/coreComponents/physicsSolvers/fluidFlow/FlowSolverBase.cpp
@@ -671,15 +671,13 @@ void FlowSolverBase::findMinMaxElevationInEquilibriumTarget( DomainPartition & d
} );
- MpiWrapper::allReduce( localMaxElevation.data(),
- maxElevation.data(),
- localMaxElevation.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Max ),
+ MpiWrapper::allReduce( localMaxElevation.toView(),
+ maxElevation,
+ MpiWrapper::Reduction::Max,
MPI_COMM_GEOS );
- MpiWrapper::allReduce( localMinElevation.data(),
- minElevation.data(),
- localMinElevation.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Min ),
+ MpiWrapper::allReduce( localMinElevation.toView(),
+ minElevation,
+ MpiWrapper::Reduction::Min,
MPI_COMM_GEOS );
}
@@ -726,10 +724,9 @@ void FlowSolverBase::computeSourceFluxSizeScalingFactor( real64 const & time,
} );
// synchronize the set size over all the MPI ranks
- MpiWrapper::allReduce( bcAllSetsSize.data(),
- bcAllSetsSize.data(),
- bcAllSetsSize.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+ MpiWrapper::allReduce( bcAllSetsSize,
+ bcAllSetsSize,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
}
@@ -809,10 +806,9 @@ void FlowSolverBase::saveAquiferConvergedState( real64 const & time,
localSumFluxes[aquiferIndex] += targetSetSumFluxes;
} );
- MpiWrapper::allReduce( localSumFluxes.data(),
- globalSumFluxes.data(),
- localSumFluxes.size(),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+ MpiWrapper::allReduce( localSumFluxes,
+ globalSumFluxes,
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
// Step 3: we are ready to save the summed fluxes for each individual aquifer
diff --git a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp
index dbf00bbb3d7..3175f518666 100644
--- a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp
+++ b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp
@@ -226,11 +226,9 @@ real64 HydrofractureSolver< POROMECHANICS_SOLVER >::fullyCoupledSolverStep( real
{
locallyFractured = 1;
}
- MpiWrapper::allReduce( &locallyFractured,
- &globallyFractured,
- 1,
- MPI_MAX,
- MPI_COMM_GEOS );
+ globallyFractured = MpiWrapper::allReduce( locallyFractured,
+ MpiWrapper::Reduction::Max,
+ MPI_COMM_GEOS );
if( globallyFractured == 0 )
{
diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp
index fa5c1685db3..dd19b20d9b3 100644
--- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp
+++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp
@@ -513,11 +513,9 @@ real64 SolidMechanicsLagrangianFEM::solverStep( real64 const & time_n,
{
locallyFractured = 1;
}
- MpiWrapper::allReduce( &locallyFractured,
- &globallyFractured,
- 1,
- MPI_MAX,
- MPI_COMM_GEOS );
+ globallyFractured = MpiWrapper::allReduce( locallyFractured,
+ MpiWrapper::Reduction::Max,
+ MPI_COMM_GEOS );
}
if( globallyFractured == 0 )
{
diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp
index a2b1d3ec2ff..c576f6513e2 100644
--- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp
+++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp
@@ -3313,23 +3313,17 @@ void SolidMechanicsMPM::printProfilingResults()
// Get total CPU time for the entire time step
real64 totalStepTimeThisRank = m_profilingTimes[numIntervals] - m_profilingTimes[0];
- real64 totalStepTimeAllRanks;
- MpiWrapper::allReduce< real64 >( &totalStepTimeThisRank,
- &totalStepTimeAllRanks,
- 1,
- MPI_SUM,
- MPI_COMM_GEOS );
+ real64 const totalStepTimeAllRanks = MpiWrapper::allReduce( totalStepTimeThisRank,
+ MpiWrapper::Reduction::Sum,
+ MPI_COMM_GEOS );
// Get total CPU times for each queried time interval
for( unsigned int i = 0; i < numIntervals; i++ )
{
real64 timeIntervalThisRank = ( m_profilingTimes[i+1] - m_profilingTimes[i] );
- real64 timeIntervalAllRanks;
- MpiWrapper::allReduce< real64 >( &timeIntervalThisRank,
- &timeIntervalAllRanks,
- 1,
- MPI_SUM,
- MPI_COMM_GEOS );
+ real64 const timeIntervalAllRanks = MpiWrapper::allReduce( timeIntervalThisRank,
+ MpiWrapper::Reduction::Sum,
+ MPI_COMM_GEOS );
if( rank == 0 )
{
timeIntervalsAllRanks[i] = timeIntervalAllRanks;
diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsStatistics.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsStatistics.cpp
index d16dd1ea519..12f47f2b3e3 100644
--- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsStatistics.cpp
+++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsStatistics.cpp
@@ -146,16 +146,14 @@ void SolidMechanicsStatistics::computeNodeStatistics( MeshLevel & mesh, real64 c
nodeStatistics.minDisplacement[1] = minDispY.get();
nodeStatistics.minDisplacement[2] = minDispZ.get();
- MpiWrapper::allReduce( nodeStatistics.maxDisplacement.data(),
- nodeStatistics.maxDisplacement.data(),
- 3,
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Max ),
+ MpiWrapper::allReduce( nodeStatistics.maxDisplacement,
+ nodeStatistics.maxDisplacement,
+ MpiWrapper::Reduction::Max,
MPI_COMM_GEOS );
- MpiWrapper::allReduce( nodeStatistics.minDisplacement.data(),
- nodeStatistics.minDisplacement.data(),
- 3,
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Min ),
+ MpiWrapper::allReduce( nodeStatistics.minDisplacement,
+ nodeStatistics.minDisplacement,
+ MpiWrapper::Reduction::Min,
MPI_COMM_GEOS );
TableData mechanicsData;
diff --git a/src/coreComponents/physicsSolvers/surfaceGeneration/SurfaceGenerator.cpp b/src/coreComponents/physicsSolvers/surfaceGeneration/SurfaceGenerator.cpp
index 788c078ada3..e41674c9f82 100644
--- a/src/coreComponents/physicsSolvers/surfaceGeneration/SurfaceGenerator.cpp
+++ b/src/coreComponents/physicsSolvers/surfaceGeneration/SurfaceGenerator.cpp
@@ -4580,12 +4580,9 @@ SurfaceGenerator::calculateRuptureRate( SurfaceElementRegion & faceElementRegion
maxRuptureRate = std::max( maxRuptureRate, ruptureRate( faceElemIndex ) );
}
- real64 globalMaxRuptureRate;
- MpiWrapper::allReduce( &maxRuptureRate,
- &globalMaxRuptureRate,
- 1,
- MPI_MAX,
- MPI_COMM_GEOS );
+ real64 const globalMaxRuptureRate = MpiWrapper::allReduce( maxRuptureRate,
+ MpiWrapper::Reduction::Max,
+ MPI_COMM_GEOS );
return globalMaxRuptureRate;
}
diff --git a/src/coreComponents/physicsSolvers/wavePropagation/sem/elastic/secondOrderEqn/isotropic/ElasticWaveEquationSEM.cpp b/src/coreComponents/physicsSolvers/wavePropagation/sem/elastic/secondOrderEqn/isotropic/ElasticWaveEquationSEM.cpp
index acf9572b216..84277473919 100644
--- a/src/coreComponents/physicsSolvers/wavePropagation/sem/elastic/secondOrderEqn/isotropic/ElasticWaveEquationSEM.cpp
+++ b/src/coreComponents/physicsSolvers/wavePropagation/sem/elastic/secondOrderEqn/isotropic/ElasticWaveEquationSEM.cpp
@@ -1013,10 +1013,10 @@ void ElasticWaveEquationSEM::cleanup( real64 const time_n,
computeAllSeismoTraces( time_n, 0.0, uy_np1, uy_n, dasReceivers, m_linearDASVectorY.toView(), true );
computeAllSeismoTraces( time_n, 0.0, uz_np1, uz_n, dasReceivers, m_linearDASVectorZ.toView(), true );
// sum contributions from all MPI ranks, since some receivers might be split among multiple ranks
- MpiWrapper::allReduce( dasReceivers.data(),
- dasReceivers.data(),
+ MpiWrapper::allReduce( dasReceivers,
+ dasReceivers,
m_linearDASGeometry.size( 0 ),
- MpiWrapper::getMpiOp( MpiWrapper::Reduction::Sum ),
+ MpiWrapper::Reduction::Sum,
MPI_COMM_GEOS );
WaveSolverUtils::writeSeismoTrace( "dasTraceReceiver", getName(), m_outputSeismoTrace, m_linearDASGeometry.size( 0 ),
m_receiverIsLocal, m_nsamplesSeismoTrace, dasReceivers );