diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7858ab8a0..8c52d594f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,11 +17,12 @@ jobs: download-build-test: strategy: matrix: - accelerator: [cpu, cuda, rocm] + accelerator: [cpu, cuda, rocm, oneapi] real-precision: [real32, real64] mode: [debug, optimize] exclude: - {accelerator: rocm, real-precision: real32} + - {accelerator: oneapi, real-precision: real32} - {real-precision: real32, mode: optimize} env: ACCELERATOR: ${{matrix.accelerator}} @@ -36,13 +37,13 @@ jobs: steps: # Check out the repository under $GITHUB_WORKSPACE, so our job can access it - uses: actions/checkout@v4 - + # # Check out the documentation and test results # - uses: actions/checkout@v3 # with: # ref: gh-pages # path: gh-pages - + - name: Log the triggering commit run: echo ${{github.event.client_payload.trigger}} @@ -51,12 +52,14 @@ jobs: - name: Build Cactus run: /bin/bash $GITHUB_WORKSPACE/scripts/build.sh - + # The oneAPI setup does not build yet + continue-on-error: ${{matrix.accelerator == 'oneapi'}} + - name: Test Cactus - # We cannot run with cuda nor with rocm. Debug builds run too slowly. - if: ${{matrix.accelerator != 'cuda' && matrix.accelerator != 'rocm' && matrix.mode != 'debug'}} + # We cannot run on accelerators. Debug builds run too slowly. + if: ${{matrix.accelerator == 'cpu' && matrix.mode == 'optimize'}} run: /bin/bash $GITHUB_WORKSPACE/scripts/test.sh - + # Set exit code based on test status - name: Set exit code # Only look at the test results for `carpetx-cpu-real64` runs diff --git a/Algo/src/roots.cxx b/Algo/src/roots.cxx index 99b084ef1..fcd532404 100644 --- a/Algo/src/roots.cxx +++ b/Algo/src/roots.cxx @@ -138,7 +138,16 @@ extern "C" void Test_roots(CCTK_ARGUMENTS) { const int maxiters = 100; int iters; bool failed; - auto x = newton_raphson_nd(gnd, vec{1.0, 1.0}, + + // SYCL cannot call the templated function above, so we define a + // local copy + auto gnd_CCTK_REAL = [](vec x) + -> std::pair, mat > { + return {vec{x(0) * x(0) - 2, x(0) * x(1) - 2}, + mat{2 * x(0), x(1), 0, x(0)}}; + }; + + auto x = newton_raphson_nd(gnd_CCTK_REAL, vec{1.0, 1.0}, vec{0.0, 0.0}, vec{10.0, 10.0}, minbits, maxiters, iters, failed); diff --git a/TestSymmetries/src/init.cxx b/TestSymmetries/src/init.cxx index 9d14a708a..81c56e3ce 100644 --- a/TestSymmetries/src/init.cxx +++ b/TestSymmetries/src/init.cxx @@ -275,7 +275,7 @@ extern "C" void TestSymmetries_Check(CCTK_ARGUMENTS) { auto &var, const auto &value) CCTK_ATTRIBUTE_ALWAYS_INLINE { using std::abs; if (abs(var - value) > 10 * std::numeric_limits::epsilon()) { -#if !defined __CUDACC__ && !defined __HIPCC__ +#if !defined __CUDACC__ && !defined __HIPCC__ && !defined SYCL_LANGUAGE_VERSION CCTK_VERROR( "Grid function symmetry check failed: I=[%d,%d,%d] X=[%g,%g,%g] " "centering=[%d,%d,%d] parity=[%d,%d,%d] var=%.17g value=%.17g", diff --git a/docker/carpetx-arm64v8-cpu.dockerfile b/docker/carpetx-arm64v8-cpu.dockerfile index 1a71b0fb8..4f994f4af 100644 --- a/docker/carpetx-arm64v8-cpu.dockerfile +++ b/docker/carpetx-arm64v8-cpu.dockerfile @@ -7,8 +7,8 @@ # docker push einsteintoolkit/carpetx:arm64v8-cpu-real32 # noble is ubuntu:24.04 -# FROM arm64v8/ubuntu:noble-20240423 -FROM arm64v8/ubuntu:noble-20240429 +# FROM arm64v8/ubuntu:noble-20240429 +FROM arm64v8/ubuntu:noble-20240530 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ diff --git a/docker/carpetx-cpu.dockerfile b/docker/carpetx-cpu.dockerfile index 47de29f97..dbfb12620 100644 --- a/docker/carpetx-cpu.dockerfile +++ b/docker/carpetx-cpu.dockerfile @@ -7,8 +7,8 @@ # docker push einsteintoolkit/carpetx:cpu-real32 # noble is ubuntu:24.04 -# FROM ubuntu:noble-20240423 -FROM ubuntu:noble-20240429 +# FROM ubuntu:noble-20240429 +FROM ubuntu:noble-20240530 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ diff --git a/docker/carpetx-oneapi.dockerfile b/docker/carpetx-oneapi.dockerfile index 813f2188b..5eda62141 100644 --- a/docker/carpetx-oneapi.dockerfile +++ b/docker/carpetx-oneapi.dockerfile @@ -6,8 +6,8 @@ # docker build --build-arg real_precision=real32 --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real32 . # docker push einsteintoolkit/carpetx:oneapi-real32 -# FROM intel/oneapi-basekit:devel-ubuntu22.04 -FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04 +# FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04 +FROM intel/oneapi-basekit:2024.1.1-devel-ubuntu22.04 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -66,6 +66,18 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 28DA432DAAC8BAEA && && \ rm -rf /var/lib/apt/lists/* +# Remove troublesome libraries +RUN find /opt/intel -name 'impi.pc' -delete && \ + find /opt/intel -name 'libhwloc.a' -delete && \ + find /opt/intel -name 'libhwloc.so' -delete && \ + find /opt/intel -name 'libmpi*a' -delete && \ + find /opt/intel -name 'libmpi*so' -delete && \ + find /opt/intel -name 'mpi.h' -delete && \ + find /opt/intel -name 'mpicc' -delete && \ + find /opt/intel -name 'mpicxx' -delete && \ + find /opt/intel -name 'mpiexec' -delete && \ + find /opt/intel -name 'mpirun' -delete + # Install blosc2 # blosc2 is a compression library, comparable to zlib RUN mkdir src && \ @@ -247,10 +259,10 @@ ARG real_precision=real64 # Should we keep the AMReX source tree around for debugging? RUN mkdir src && \ (cd src && \ - wget https://github.com/AMReX-Codes/amrex/archive/24.05.tar.gz && \ - tar xzf 24.05.tar.gz && \ + wget https://github.com/AMReX-Codes/amrex/archive/24.06.tar.gz && \ + tar xzf 24.06.tar.gz && \ rm -rf /opt/intel/oneapi/mpi && \ - cd amrex-24.05 && \ + cd amrex-24.06 && \ case $real_precision in \ real32) precision=SINGLE;; \ real64) precision=DOUBLE;; \ diff --git a/docker/carpetx-rocm.dockerfile b/docker/carpetx-rocm.dockerfile index 1cd1ae408..ef95f8a83 100644 --- a/docker/carpetx-rocm.dockerfile +++ b/docker/carpetx-rocm.dockerfile @@ -6,8 +6,8 @@ # docker build --build-arg real_precision=real32 --file carpetx-rocm.dockerfile --tag einsteintoolkit/carpetx:rocm-real32 . # docker push einsteintoolkit/carpetx:rocm-real32 -# FROM rocm/dev-ubuntu-22.04:6.1 -FROM rocm/dev-ubuntu-22.04:6.1.1 +# FROM rocm/dev-ubuntu-22.04:6.1.1 +FROM rocm/dev-ubuntu-22.04:6.1.2 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ diff --git a/scripts/actions-cpu-real32.cfg b/scripts/actions-cpu-real32.cfg index 6aea09575..c4115068f 100644 --- a/scripts/actions-cpu-real32.cfg +++ b/scripts/actions-cpu-real32.cfg @@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD # REPRIMAND_LIBS = RePrimAnd # RNPLETAL_DIR = /usr/local SILO_DIR = /usr/local -# SIMULATIONIO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local SSHT_DIR = /usr/local YAML_CPP_DIR = /usr/local ZLIB_DIR = /usr diff --git a/scripts/actions-cpu-real64.cfg b/scripts/actions-cpu-real64.cfg index c99699f44..dfc3733a7 100644 --- a/scripts/actions-cpu-real64.cfg +++ b/scripts/actions-cpu-real64.cfg @@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD # REPRIMAND_LIBS = RePrimAnd # RNPLETAL_DIR = /usr/local SILO_DIR = /usr/local -# SIMULATIONIO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local SSHT_DIR = /usr/local YAML_CPP_DIR = /usr/local ZLIB_DIR = /usr diff --git a/scripts/actions-cuda-real32.cfg b/scripts/actions-cuda-real32.cfg index f035811ea..46b94a970 100644 --- a/scripts/actions-cuda-real32.cfg +++ b/scripts/actions-cuda-real32.cfg @@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD # REPRIMAND_LIBS = RePrimAnd # RNPLETAL_DIR = /usr/local SILO_DIR = /usr/local -# SIMULATIONIO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local SSHT_DIR = /usr/local YAML_CPP_DIR = /usr/local ZLIB_DIR = /usr diff --git a/scripts/actions-cuda-real64.cfg b/scripts/actions-cuda-real64.cfg index 889234f65..f81beb5dd 100644 --- a/scripts/actions-cuda-real64.cfg +++ b/scripts/actions-cuda-real64.cfg @@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD # REPRIMAND_LIBS = RePrimAnd # RNPLETAL_DIR = /usr/local SILO_DIR = /usr/local -# SIMULATIONIO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local SSHT_DIR = /usr/local YAML_CPP_DIR = /usr/local ZLIB_DIR = /usr diff --git a/scripts/actions-oneapi-real64.cfg b/scripts/actions-oneapi-real64.cfg new file mode 100644 index 000000000..274733599 --- /dev/null +++ b/scripts/actions-oneapi-real64.cfg @@ -0,0 +1,102 @@ +# Option list for the Einstein Toolkit + +# The "weird" options here should probably be made the default in the +# ET instead of being set here. + +# Whenever this version string changes, the application is configured +# and rebuilt from scratch +VERSION = actions-oneapi-real64-2023-04-11 + +CPP = cpp +CC = /opt/intel/oneapi/compiler/2024.1/bin/icx +CXX = /opt/intel/oneapi/compiler/2024.1/bin/icpx +FPP = cpp +FC = gfortran +F90 = gfortran +LD = /opt/intel/oneapi/compiler/2024.1/bin/icpx + +CPPFLAGS = -DSIMD_CPU +CFLAGS = -fp-model=precise -g -march=x86-64-v3 -pipe -std=gnu11 +CXXFLAGS = -fp-model=precise -fsycl -g -march=x86-64-v3 -pipe -std=c++17 +FPPFLAGS = -traditional +F90FLAGS = -fcray-pointer -ffixed-line-length-none -g -march=x86-64-v3 -pipe + +SYS_INC_DIRS = /opt/intel/oneapi/2024.1/include +LIBDIRS = /usr/local/lib +LIBS = gfortran + +C_LINE_DIRECTIVES = yes +F_LINE_DIRECTIVES = yes + +INTEGER_PRECISION = 4 +REAL_PRECISION = 8 + +# oneAPI does not support these +# DISABLE_INT16 = yes +DISABLE_REAL16 = yes + +DEBUG = no +CPP_DEBUG_FLAGS = -DCARPET_DEBUG +C_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv +CXX_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv +FPP_DEBUG_FLAGS = -DCARPET_DEBUG +F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -ftrapv + +OPTIMISE = yes +C_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math +CXX_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math +F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans + +# Clang segfaults with OpenMP enabled +OPENMP = no +CPP_OPENMP_FLAGS = -fopenmp +FPP_OPENMP_FLAGS = -D_OPENMP +F90_OPENMP_FLAGS = -fopenmp + +WARN = yes +CPP_WARN_FLAGS = -Wall +C_WARN_FLAGS = +CXX_WARN_FLAGS = +FPP_WARN_FLAGS = -Wall +F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising + +VECTORISE = no + +ADIOS2_DIR = /usr/local +ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 +AMREX_DIR = /usr/local +# ASDF_CXX_DIR = /usr/local +BLAS_DIR = /usr +BOOST_DIR = /usr +FFTW3_DIR = /usr +GSL_DIR = /usr +HDF5_DIR = /usr/lib/x86_64-linux-gnu/hdf5/serial +HDF5_ENABLE_CXX = yes +HDF5_ENABLE_FORTRAN = yes +HDF5_INC_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/include +HDF5_LIB_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/lib +HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5 +HWLOC_DIR = /usr +# JEMALLOC_DIR = /usr/local +LAPACK_DIR = /usr +# LIBJPEG_DIR = /usr +# LORENE_DIR = /usr/local +MPI_DIR = /usr +NSIMD_DIR = /usr/local +NSIMD_SIMD = AVX2 +OPENBLAS_DIR = /usr +OPENPMD_API_DIR = /usr/local +# OPENSSL_DIR = /usr +# PAPI_DIR = /usr/local +PETSC_DIR = /usr/lib/petsc +PETSC_LIBS = petsc_real +PETSC_ARCH_LIBS = m +PTHREADS_DIR = NO_BUILD +# REPRIMAND_DIR = /usr/local +# REPRIMAND_LIBS = RePrimAnd +# RNPLETAL_DIR = /usr/local +SILO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local +SSHT_DIR = /usr/local +YAML_CPP_DIR = /usr/local +ZLIB_DIR = /usr diff --git a/scripts/actions-oneapi-real64.ini b/scripts/actions-oneapi-real64.ini new file mode 100644 index 000000000..0a2303ad9 --- /dev/null +++ b/scripts/actions-oneapi-real64.ini @@ -0,0 +1,72 @@ +[actions-oneapi-real64] + +# Machine description +nickname = actions-oneapi-real64 +name = Actions +location = Github +description = Github action for CarpetX CI +webpage = https://github.com/eschnett/CarpetX +status = production + +# Access to this machine +hostname = actions +aliaspattern = ^actions$ +# envsetup = </dev/null 2>/dev/null & echo $! +submit = @SCRIPTFILE@ /dev/null 2>/dev/null; true & echo $! +getstatus = ps @JOB_ID@ +stop = kill @JOB_ID@ +submitpattern = (.*) +statuspattern = "^ *@JOB_ID@ " +queuedpattern = $^ +runningpattern = ^ +holdingpattern = $^ +exechost = echo localhost +exechostpattern = (.*) +stdout = cat @SIMULATION_NAME@.out +stderr = cat @SIMULATION_NAME@.err +stdout-follow = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err diff --git a/scripts/actions-oneapi-real64.run b/scripts/actions-oneapi-real64.run new file mode 100644 index 000000000..3921feb6e --- /dev/null +++ b/scripts/actions-oneapi-real64.run @@ -0,0 +1,45 @@ +#! /bin/bash + +echo "Preparing:" +set -x # Output commands +set -e # Abort on errors + +cd @RUNDIR@-active + +echo "Checking:" +pwd +hostname +date + +echo "Environment:" +export "SIMULATION_ID=@SIMULATION_ID@" +# OpenMP +export CACTUS_NUM_PROCS=@NUM_PROCS@ +export CACTUS_NUM_THREADS=@NUM_THREADS@ +export CACTUS_SET_THREAD_BINDINGS=0 +export GLIBCXX_FORCE_NEW=1 +export GMON_OUT_PREFIX=gmon.out +export OMP_DISPLAY_ENV=TRUE +export OMP_MAX_TASKS=100 +export OMP_NUM_THREADS=@NUM_THREADS@ +export OMP_PLACES=cores # threads, cores, sockets +export OMP_PROC_BIND=FALSE # false, true, master, close, spread +export OMP_STACKSIZE=8192 # kByte +env | sort >SIMFACTORY/ENVIRONMENT + +# ulimit -c unlimited + +echo "Starting:" +export CACTUS_STARTTIME=$(date +%s) + +time \ + mpiexec \ + -n @NUM_PROCS@ \ + @EXECUTABLE@ -L 3 @PARFILE@ \ + >stdout.txt \ + 2>stderr.txt + +echo "Stopping:" +date + +echo "Done." diff --git a/scripts/actions-oneapi-real64.sub b/scripts/actions-oneapi-real64.sub new file mode 100644 index 000000000..cd7ec3a81 --- /dev/null +++ b/scripts/actions-oneapi-real64.sub @@ -0,0 +1,3 @@ +#! /bin/bash +cd @SOURCEDIR@ +exec @SIMFACTORY@ run @SIMULATION_NAME@ --machine=@MACHINE@ --restart-id=@RESTART_ID@ @FROM_RESTART_COMMAND@ @RUNDIR@/stdlog.txt 2>&1 diff --git a/scripts/actions-rocm-real64.cfg b/scripts/actions-rocm-real64.cfg index 7b8bb214f..1fe8a0ecf 100644 --- a/scripts/actions-rocm-real64.cfg +++ b/scripts/actions-rocm-real64.cfg @@ -97,7 +97,7 @@ PTHREADS_DIR = NO_BUILD # REPRIMAND_LIBS = RePrimAnd # RNPLETAL_DIR = /usr/local SILO_DIR = /usr/local -# SIMULATIONIO_DIR = /usr/local +SIMULATIONIO_DIR = /usr/local SSHT_DIR = /usr/local YAML_CPP_DIR = /usr/local ZLIB_DIR = /usr