From 20e7e9feb2684818ba2db2ba0c3b03cb4b178c62 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 30 Mar 2024 19:47:10 -0400 Subject: [PATCH] CI: Add oneAPI setup Also update package version. --- .github/workflows/ci.yml | 7 +- docker/carpetx-cpu.dockerfile | 16 +- docker/carpetx-cuda.dockerfile | 12 +- docker/carpetx-oneapi.dockerfile | 266 +++++++++++++++++++++++++++++++ docker/carpetx-rocm.dockerfile | 12 +- 5 files changed, 290 insertions(+), 23 deletions(-) create mode 100644 docker/carpetx-oneapi.dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b236c910b..2ca8bbc8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,11 +17,12 @@ jobs: download-build-test: strategy: matrix: - accelerator: [cpu, cuda, rocm] + accelerator: [cpu, cuda, rocm, oneapi] real-precision: [real32, real64] mode: [debug, optimize] exclude: - {accelerator: rocm, real-precision: real32} + - {accelerator: oneapi, real-precision: real32} - {real-precision: real32, mode: optimize} env: ACCELERATOR: ${{matrix.accelerator}} @@ -53,8 +54,8 @@ jobs: run: /bin/bash $GITHUB_WORKSPACE/scripts/build.sh - name: Test Cactus - # We cannot run with cuda nor with rocm. Debug builds run too slowly. - if: ${{matrix.accelerator != 'cuda' && matrix.accelerator != 'rocm' && matrix.mode != 'debug'}} + # We cannot run on accelerators. Debug builds run too slowly. + if: ${{matrix.accelerator == 'cpu' && matrix.mode == 'optimize'}} run: /bin/bash $GITHUB_WORKSPACE/scripts/test.sh # Set exit code based on test status diff --git a/docker/carpetx-cpu.dockerfile b/docker/carpetx-cpu.dockerfile index 897bb0295..9399f8172 100644 --- a/docker/carpetx-cpu.dockerfile +++ b/docker/carpetx-cpu.dockerfile @@ -7,8 +7,8 @@ # docker push einsteintoolkit/carpetx:cpu-real32 # jammy is ubuntu:22.04 -# FROM ubuntu:jammy-20240125 -FROM ubuntu:jammy-20240212 +# FROM ubuntu:jammy-20240212 +FROM ubuntu:jammy-20240227 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -103,9 +103,9 @@ RUN apt-get update && \ # blosc2 is a compression library, comparable to zlib RUN mkdir src && \ (cd src && \ - wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.1.tar.gz && \ - tar xzf v2.13.1.tar.gz && \ - cd c-blosc2-2.13.1 && \ + wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.2.tar.gz && \ + tar xzf v2.13.2.tar.gz && \ + cd c-blosc2-2.13.2 && \ cmake -B build -G Ninja -S . \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -261,9 +261,9 @@ ARG real_precision=real64 # Should we keep the AMReX source tree around for debugging? RUN mkdir src && \ (cd src && \ - wget https://github.com/AMReX-Codes/amrex/archive/24.02.tar.gz && \ - tar xzf 24.02.tar.gz && \ - cd amrex-24.02 && \ + wget https://github.com/AMReX-Codes/amrex/archive/24.03.tar.gz && \ + tar xzf 24.03.tar.gz && \ + cd amrex-24.03 && \ case $real_precision in \ real32) precision=SINGLE;; \ real64) precision=DOUBLE;; \ diff --git a/docker/carpetx-cuda.dockerfile b/docker/carpetx-cuda.dockerfile index 85126ed19..19367a575 100644 --- a/docker/carpetx-cuda.dockerfile +++ b/docker/carpetx-cuda.dockerfile @@ -103,9 +103,9 @@ RUN apt-get update && \ # blosc2 is a compression library, comparable to zlib RUN mkdir src && \ (cd src && \ - wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.1.tar.gz && \ - tar xzf v2.13.1.tar.gz && \ - cd c-blosc2-2.13.1 && \ + wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.2.tar.gz && \ + tar xzf v2.13.2.tar.gz && \ + cd c-blosc2-2.13.2 && \ cmake -B build -G Ninja -S . \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -261,9 +261,9 @@ ARG real_precision=real64 # Should we keep the AMReX source tree around for debugging? RUN mkdir src && \ (cd src && \ - wget https://github.com/AMReX-Codes/amrex/archive/24.02.tar.gz && \ - tar xzf 24.02.tar.gz && \ - cd amrex-24.02 && \ + wget https://github.com/AMReX-Codes/amrex/archive/24.03.tar.gz && \ + tar xzf 24.03.tar.gz && \ + cd amrex-24.03 && \ case $real_precision in \ real32) precision=SINGLE;; \ real64) precision=DOUBLE;; \ diff --git a/docker/carpetx-oneapi.dockerfile b/docker/carpetx-oneapi.dockerfile new file mode 100644 index 000000000..f523b1a9c --- /dev/null +++ b/docker/carpetx-oneapi.dockerfile @@ -0,0 +1,266 @@ + +# How to build this Docker image: + +# docker build --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real64 . +# docker push einsteintoolkit/carpetx:oneapi-real64 + +# docker build --build-arg real_precision=real32 --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real32 . +# docker push einsteintoolkit/carpetx:oneapi-real32 + +# FROM intel/oneapi-basekit:devel-ubuntu22.04 +FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANGUAGE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +RUN mkdir /cactus +WORKDIR /cactus + +# Install system packages +# - Boost on Ubuntu requires OpenMPI +RUN apt-get update && \ + apt-get --yes --no-install-recommends install \ + ca-certificates \ + cmake \ + cvs \ + diffutils \ + g++ \ + gcc \ + gdb \ + gfortran \ + git \ + hdf5-tools \ + language-pack-en \ + less \ + libblosc-dev \ + libboost-all-dev \ + libbz2-dev \ + libfftw3-dev \ + libgit2-dev \ + libgsl-dev \ + libhdf5-dev \ + libhwloc-dev \ + libopenblas-dev \ + libopenmpi-dev \ + libpetsc-real-dev \ + libtool \ + libudev-dev \ + libyaml-cpp-dev \ + locales \ + m4 \ + meson \ + ninja-build \ + numactl \ + perl \ + pkgconf \ + python3 \ + python3-pip \ + python3-requests \ + rsync \ + subversion \ + vim \ + wget \ + zlib1g-dev \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install blosc2 +# blosc2 is a compression library, comparable to zlib +RUN mkdir src && \ + (cd src && \ + wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.2.tar.gz && \ + tar xzf v2.13.2.tar.gz && \ + cd c-blosc2-2.13.2 && \ + cmake -B build -G Ninja -S . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_BENCHMARKS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_TESTS=OFF \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install ADIOS2 +# ADIOS2 is a parallel I/O library, comparable to HDF5 +RUN mkdir src && \ + (cd src && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.2.tar.gz && \ + tar xzf v2.9.2.tar.gz && \ + cd ADIOS2-2.9.2 && \ + cmake -B build -G Ninja -S . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Fortran=OFF \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install ASDF +# ASDF is an I/O library like HDF5 +# - depends on yaml-cpp +RUN mkdir src && \ + (cd src && \ + wget https://github.com/eschnett/asdf-cxx/archive/refs/tags/version/7.3.2.tar.gz && \ + tar xzf 7.3.2.tar.gz && \ + cd asdf-cxx-version-7.3.2 && \ + cmake -B build -G Ninja -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr/local && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install NSIMD +# NSIMD allows writing explicitly SIMD-vectorized code +# Note: This assumes that the system has x86_64 CPUs +RUN mkdir src && \ + (cd src && \ + wget https://github.com/agenium-scale/nsimd/archive/refs/tags/v3.0.1.tar.gz && \ + tar xzf v3.0.1.tar.gz && \ + cd nsimd-3.0.1 && \ + cmake -B build -G Ninja -S . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -Dsimd=AVX2 \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install openPMD-api +# openPMD-api defines a standard for laying out AMR data in a file +# - depends on ADIOS2 +RUN mkdir src && \ + (cd src && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ + tar xzf 0.15.2.tar.gz && \ + cd openPMD-api-0.15.2 && \ + cmake -B build -G Ninja -S . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + -DBUILD_EXAMPLES=OFF \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install RePrimAnd +# RePrimAnd is a physics package for nuclear equations of state +RUN mkdir src && \ + (cd src && \ + wget https://github.com/wokast/RePrimAnd/archive/refs/tags/v1.6.tar.gz && \ + tar xzf v1.6.tar.gz && \ + cd RePrimAnd-1.6 && \ + meson build --buildtype=release --prefix=/usr/local && \ + ninja -C build && \ + ninja -C build install && \ + true) && \ + rm -rf src + +# Install Silo +# Silo defines a standard for laying out AMR data in a file +RUN mkdir src && \ + (cd src && \ + wget https://github.com/LLNL/Silo/releases/download/4.11.1/silo-4.11.1.tar.xz && \ + tar xJf silo-4.11.1.tar.xz && \ + cd silo-4.11.1 && \ + mkdir build && \ + cd build && \ + ../configure \ + --disable-fortran \ + --enable-optimization \ + --with-hdf5=/usr/lib/x86_64-linux-gnu/hdf5/serial/include,/usr/lib/x86_64-linux-gnu/hdf5/serial/lib \ + --prefix=/usr/local \ + && \ + make -j$(nproc) && \ + make -j$(nproc) install && \ + true) && \ + rm -rf src + +# Install SimulationIO +# SimulationIO is an I/O library like HDF5 +# - depends on asdf-cxx +# - depends on yaml-cpp +# Currently disabling ASDF because there is a confusion with C++11/17 standards +RUN mkdir src && \ + (cd src && \ + wget https://github.com/eschnett/SimulationIO/archive/refs/tags/version/9.0.3.tar.gz && \ + tar xzf 9.0.3.tar.gz && \ + cd SimulationIO-version-9.0.3 && \ + cmake -B build -G Ninja -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_ASDF_CXX=OFF && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +# Install ssht +# ssht provides spin-weighted spherical harmonics +RUN mkdir src && \ + (cd src && \ + wget https://github.com/astro-informatics/ssht/archive/v1.5.2.tar.gz && \ + tar xzf v1.5.2.tar.gz && \ + cd ssht-1.5.2 && \ + cmake -B build -G Ninja -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr/local && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + +ARG real_precision=real64 + +# Install AMReX +# AMReX provides adaptive mesh refinement +# - Enable Fortran for `docker/Dockerfile` +# - Install this last because it changes most often +# Should we keep the AMReX source tree around for debugging? +RUN mkdir src && \ + (cd src && \ + wget https://github.com/AMReX-Codes/amrex/archive/24.03.tar.gz && \ + tar xzf 24.03.tar.gz && \ + rm -rf /opt/intel/oneapi/mpi && \ + cd amrex-24.03 && \ + case $real_precision in \ + real32) precision=SINGLE;; \ + real64) precision=DOUBLE;; \ + *) exit 1;; \ + esac && \ + cmake -B build -G Ninja -S . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DCMAKE_PREFIX_PATH='/opt/intel/oneapi' \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_SHARED_LIBS=ON \ + -DAMReX_FORTRAN=OFF \ + -DAMReX_FORTRAN_INTERFACES=OFF \ + -DAMReX_GPU_BACKEND=SYCL \ + -DAMReX_INTEL_ARCH=intel_gpu_pvc \ + -DAMReX_OMP=OFF \ + -DAMReX_PARTICLES=ON \ + -DAMReX_PRECISION="$precision" \ + -DMPI_CXX_ADDITIONAL_INCLUDE_DIRS='/usr/lib/x86_64-linux-gnu/openmpi/include/openmpi;/usr/lib/x86_64-linux-gnu/openmpi/include' \ + -DMPI_CXX_LIB_NAMES='mpi_cxx;mpi;open-rte;open-pal;hwloc' \ + -DMPI_C_ADDITIONAL_INCLUDE_DIRS='/usr/lib/x86_64-linux-gnu/openmpi/include/openmpi;/usr/lib/x86_64-linux-gnu/openmpi/include' \ + -DMPI_C_LIB_NAMES='mpi;open-rte;open-pal;hwloc' \ + -DMPI_hwloc_LIBRARY=/lib/x86_64-linux-gnu/libhwloc.so \ + -DMPI_mpi_LIBRARY=/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so \ + -DMPI_mpi_cxx_LIBRARY=/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi_cxx.so \ + -DMPI_open-pal_LIBRARY=/lib/x86_64-linux-gnu/libopen-pal.so \ + -DMPI_open-rte_LIBRARY=/lib/x86_64-linux-gnu/libopen-rte.so \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src diff --git a/docker/carpetx-rocm.dockerfile b/docker/carpetx-rocm.dockerfile index 4021fd3e5..f4b5498eb 100644 --- a/docker/carpetx-rocm.dockerfile +++ b/docker/carpetx-rocm.dockerfile @@ -107,9 +107,9 @@ RUN apt-get update && \ # blosc2 is a compression library, comparable to zlib RUN mkdir src && \ (cd src && \ - wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.1.tar.gz && \ - tar xzf v2.13.1.tar.gz && \ - cd c-blosc2-2.13.1 && \ + wget https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.13.2.tar.gz && \ + tar xzf v2.13.2.tar.gz && \ + cd c-blosc2-2.13.2 && \ cmake -B build -G Ninja -S . \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -264,9 +264,9 @@ ARG real_precision=real64 # Should we keep the AMReX source tree around for debugging? RUN mkdir src && \ (cd src && \ - wget https://github.com/AMReX-Codes/amrex/archive/24.02.tar.gz && \ - tar xzf 24.02.tar.gz && \ - cd amrex-24.02 && \ + wget https://github.com/AMReX-Codes/amrex/archive/24.03.tar.gz && \ + tar xzf 24.03.tar.gz && \ + cd amrex-24.03 && \ case $real_precision in \ real32) precision=SINGLE;; \ real64) precision=DOUBLE;; \