CP2K 2024.1 (GH200) (#95)

* for GH200 * deployed on santis and todi
eth-cscs · May 29, 2024 · 999435a · 999435a
1 parent 72d1a81
commit 999435a
Show file tree

Hide file tree

Showing 12 changed files with 687 additions and 23 deletions.
diff --git a/config.yaml b/config.yaml
@@ -72,6 +72,13 @@ uenvs:
         eiger: [zen2]
         clariden: [a100]
       develop: False
+    "2024.1":
+      recipes:
+        gh200: 2024.1/gh200
+      deploy:
+        santis: [gh200]
+        todi: [gh200]
+      develop: False
   icon-wcp:
     "v1":
       recipes:

diff --git a/docs/uenv-cp2k.md b/docs/uenv-cp2k.md
@@ -1,39 +1,58 @@
 # CP2K
 
-[CP2K](https://www.cp2k.org/) version `2023.2`.
+[CP2K] is a quantum chemistry and solid state physics software package that can perform atomistic simulations of solid state, liquid, molecular, periodic, material, crystal, and biological systems.
 
-An environment that provides the latest version of [CP2K](https://www.cp2k.org/), along with the libraries and tools required to build a different or custom version of [CP2K](https://www.cp2k.org/).
+CP2K provides a general framework for different modeling methods such as DFT using the mixed Gaussian and plane waves approaches GPW and GAPW. Supported theory levels include DFTB, LDA, GGA, MP2, RPA, semi-empirical methods (AM1, PM3, PM6, RM1, MNDO, …), and classical force fields (AMBER, CHARMM, …). CP2K can do simulations of molecular dynamics, metadynamics, Monte Carlo, Ehrenfest dynamics, vibrational analysis, core level spectroscopy, energy minimization, and transition state optimization using NEB or dimer method. See [CP2K Features] for a detailed overview.
 
-The following environment views are provided:
+## Running
 
-* `cp2k-scalapack`: CP2K, dependencies, and [ScaLAPACK](https://www.netlib.org/scalapack/) as diagonalization library
-* `cp2k-scalapack-dev`: dependencies and [ScaLAPACK](https://www.netlib.org/scalapack/)
-* `cp2k-elpa`: CP2K, dependencies, and [ELPA](https://elpa.mpcdf.mpg.de/) as diagonalization library
-* `cp2k-elpa-dev`: dependencies and [ELPA](https://elpa.mpcdf.mpg.de/)
+```bash
+uenv start <CP2K_UENV>
+uenv modules use
+module load cp2k
+```
 
-## Building a custom version of CP2K
+!!! warning
+    [COSMA] is built with GPU-aware MPI. Make sure to set `MPICH_GPU_SUPPORT_ENABLED=1` when running [CP2K].
 
-### Using modules
+## Building from source
 
-To build your version of CP2K do the following steps:
+The [CP2K] `uenv` provides all the dependencies required to build [CP2K] from source, with several optional features enabled. You can follow these steps to build [CP2K] from source:
 
 ```bash
-# Load the required modules
-module load [...]
-cd cp2k
+# Start uenv and load develop view
+uenv start <CP2K_UENV>
+uenv view develop
+
+# cd to CP2K source directory
+cd <PATH_TO_CP2K_SOURCE>
+
+# CMake
 mkdir build && cd build
-cmake .. -DCMAKE_BUILD_TYPE=Release -DCP2K_SCALAPACK_VENDOR=MKL -DCP2K_USE_ACCEL=cuda -DCP2K_WITH_GPU=A100
-make -j20
+CC=mpicc CXX=mpic++ FC=mpifort cmake \
+    -GNinja \
+    -DCP2K_ENABLE_REGTESTS=ON \
+    -DCP2K_USE_LIBXC=ON \
+    -DCP2K_USE_LIBINT2=ON 
+    -DCP2K_USE_SPGLIB=ON \
+    -DCP2K_USE_ELPA=ON \
+    -DCP2K_USE_SPLA=ON \
+    -DCP2K_USE_SIRIUS=ON \
+    -DCP2K_USE_COSTA=ON \
+    -DCP2K_USE_ACCEL=CUDA -DCP2K_WITH_GPU=A100 \
+    ..
+
+ninja -j 32
 ```
 
-See CP2K's [README_cmake.md](https://github.com/cp2k/cp2k/blob/master/README_cmake.md) for details.
+!!! note
 
-### Using Spack
+    `[email protected]` does not support compiling for `cuda_arch=90`. Use `-DCP2K_WITH_GPU=A100` instead.
 
-```bash
-uenv start cp2k-a100.squashfs
-export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config/
-spack install cp2k [...]
-```
+See [CP2K `README_cmake.md`](https://github.com/cp2k/cp2k/blob/master/README_cmake.md) for more details.
+
+[CP2K]: https://www.cp2k.org/
+[CP2K Features]: https://www.cp2k.org/features
+[CP2K `README_cmake.md`]: https://github.com/cp2k/cp2k/blob/master/README_cmake.md
+[COSMA]: https://github.com/eth-cscs/COSMA
 
-See Spack's [CP2K package](https://packages.spack.io/package.html?name=cp2k) for details.
diff --git a/recipes/cp2k/2024.1/gh200/compilers.yaml b/recipes/cp2k/2024.1/gh200/compilers.yaml
@@ -0,0 +1,5 @@
+bootstrap:
+  spec: gcc@12
+gcc:
+  specs:
+  - [email protected]
diff --git a/recipes/cp2k/2024.1/gh200/config.yaml b/recipes/cp2k/2024.1/gh200/config.yaml
@@ -0,0 +1,6 @@
+name: cp2k
+store: /user-environment
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: v0.22.0
+modules: True
diff --git a/recipes/cp2k/2024.1/gh200/environments.yaml b/recipes/cp2k/2024.1/gh200/environments.yaml
@@ -0,0 +1,43 @@
+cp2k:
+  compiler:
+  - toolchain: gcc
+    spec: [email protected]
+  unify: true
+  specs:
+  - cmake
+  - ninja
+  - [email protected] # Can't be newer than driver version, for DBCSR
+  - openblas threads=openmp
+  - netlib-scalapack
+  - fftw
+  - libxc ~cuda
+  - libint
+  - spglib
+  - spla
+  - elpa
+  # COSMA dependencies (need to be in the develop view)
+  - tiled-mm
+  - costa
+  - cosma +gpu_direct
+  # SIRIUS dependencies (need to be in the develop view)
+  - hdf5
+  - spfft
+  - sirius ~memory_pool
+  # cuda_arch=90 not yet supported by DBCSR and CP2K
+  - dbcsr +cuda cuda_arch=80
+  - [email protected] +libxc +libint +spglib +cosma +spla +elpa +sirius lmax=5 +cuda cuda_arch=80 build_system=cmake ^cray-mpich
+  variants:
+    - +mpi
+    - +openmp
+    - +cuda
+    - cuda_arch=90
+    - build_type=Release
+  mpi:
+    spec: cray-mpich
+    gpu: cuda
+  views:
+    develop:
+      link: roots
+      exclude: ["cp2k"]
+    cp2k:
+      link: roots
diff --git a/recipes/cp2k/2024.1/gh200/modules.yaml b/recipes/cp2k/2024.1/gh200/modules.yaml
@@ -0,0 +1,21 @@
+modules:
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'
diff --git a/recipes/cp2k/2024.1/gh200/repo/packages/cosma/fj-ssl2.patch b/recipes/cp2k/2024.1/gh200/repo/packages/cosma/fj-ssl2.patch
@@ -0,0 +1,104 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 1fd1e55..41a041b 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -19,7 +19,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS "YES") # always write compile_commands.json
+
+ set(COSMA_GPU_BACKENDS_LIST "CUDA" "ROCM")
+ set(COSMA_SCALAPACK_LIST "OFF" "MKL" "CRAY_LIBSCI" "CUSTOM")
+-set(COSMA_BLAS_LIST   "auto" "MKL" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
++set(COSMA_BLAS_LIST   "auto" "MKL" "SSL2" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
+ option(COSMA_WITH_TESTS "Generate the test target." ON)
+ option(COSMA_WITH_APPS "Generate the miniapp targets." ON)
+ option(COSMA_WITH_BENCHMARKS "Generate the benchmark targets." ON)
+@@ -45,7 +45,7 @@ if (COSMA_BLAS MATCHES "CUDA|ROCM")
+   set(COSMA_GPU_BACKEND ${COSMA_BLAS})
+ else()
+   if(COSMA_BLAS STREQUAL "OFF")
+-    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
++    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, SSL2, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
+   else()
+     set(COSMA_BLAS_VENDOR ${COSMA_BLAS})
+   endif()
+@@ -190,6 +190,7 @@ install(FILES "${cosma_BINARY_DIR}/cosmaConfig.cmake"
+   "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+   "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindMKL.cmake"
++  "${cosma_SOURCE_DIR}/cmake/FindSSL2.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindBlas.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindSCALAPACK.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindOPENBLAS.cmake"
+diff --git a/cmake/FindBlas.cmake b/cmake/FindBlas.cmake
+index aef956c..3c47561 100644
+--- a/cmake/FindBlas.cmake
++++ b/cmake/FindBlas.cmake
+@@ -14,6 +14,7 @@ endif()
+ set(COSMA_BLAS_VENDOR_LIST
+   "auto"
+   "MKL"
++  "SSL2"
+   "OPENBLAS"
+   "FLEXIBLAS"
+   "ARMPL"
+diff --git a/cmake/FindSSL2.cmake b/cmake/FindSSL2.cmake
+new file mode 100644
+index 0000000..f0e11bf
+--- /dev/null
++++ b/cmake/FindSSL2.cmake
+@@ -0,0 +1,56 @@
++#.rst:
++# FindSSL2
++# -----------
++#
++# This module tries to find the SSL2 library.
++#
++# The following variables are set
++#
++# ::
++#
++#   SSL2_FOUND           - True if ssl2 is found
++#   SSL2_LIBRARIES       - The required libraries
++#   SSL2_INCLUDE_DIRS    - The required include directory
++#
++# The following import target is created
++#
++# ::
++#
++#   SSL2::ssl2
++
++#set paths to look for library from ROOT variables.If new policy is set, find_library() automatically uses them.
++# if(NOT POLICY CMP0074)
++set(_SSL2_PATHS ${SSL2_ROOT}
++                 $ENV{SSL2_ROOT}
++                 $ENV{SSL2ROOT}
++                 $ENV{SSL2_DIR}
++                 $ENV{SSL2DIR})
++# endif()
++
++find_library(
++    COSMA_SSL2_LINK_LIBRARIES
++    NAMES "fjlapackex"
++    HINTS ${_SSL2_PATHS}
++    PATH_SUFFIXES "lib64"
++)
++find_path(
++    COSMA_SSL2_INCLUDE_DIRS
++    NAMES "cblas.h" 
++    HINTS ${_SSL2_PATHS}
++    PATH_SUFFIXES "include"
++)
++
++# check if found
++include(FindPackageHandleStandardArgs)
++find_package_handle_standard_args(SSL2 REQUIRED_VARS COSMA_SSL2_INCLUDE_DIRS COSMA_SSL2_LINK_LIBRARIES)
++
++# add target to link against
++if(NOT TARGET cosma::BLAS::SSL2::ssl2)
++  add_library(cosma::BLAS::SSL2::ssl2 INTERFACE IMPORTED)
++  add_library(cosma::BLAS::SSL2::blas ALIAS cosma::BLAS::SSL2::ssl2)
++endif()
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_LINK_LIBRARIES ${COSMA_SSL2_LINK_LIBRARIES})
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${COSMA_SSL2_INCLUDE_DIRS})
++
++# prevent clutter in cache
++MARK_AS_ADVANCED(SSL2_FOUND SSL2_LIBRARIES SSL2_INCLUDE_DIRS)
diff --git a/recipes/cp2k/2024.1/gh200/repo/packages/cosma/mpi-view.patch b/recipes/cp2k/2024.1/gh200/repo/packages/cosma/mpi-view.patch
@@ -0,0 +1,13 @@
+diff --git a/cmake/cosmaConfig.cmake.in b/cmake/cosmaConfig.cmake.in
+index cf7038f..1527c53 100644
+--- a/cmake/cosmaConfig.cmake.in
++++ b/cmake/cosmaConfig.cmake.in
+@@ -11,7 +11,7 @@ if(NOT TARGET cosma::cosma)
+     set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_MODULE_PATH})
+
+     set(MKL_ROOT "@MKL_ROOT@")
+-    set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
++    #    set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
+     set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+     find_package(MPI COMPONENTS "CXX" REQUIRED)
+