From 27c83f4c1015f6b94ce0ebf63b71770e3cf49b44 Mon Sep 17 00:00:00 2001 From: aaronkho Date: Wed, 23 Oct 2024 11:08:21 -0400 Subject: [PATCH 1/5] Added platform file for general Pixi Conda build --- platform/build/make.inc.PIXI | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 platform/build/make.inc.PIXI diff --git a/platform/build/make.inc.PIXI b/platform/build/make.inc.PIXI new file mode 100644 index 000000000..97750c263 --- /dev/null +++ b/platform/build/make.inc.PIXI @@ -0,0 +1,70 @@ +#--------------------------------------------------- +# OSX, Linux mpich using pixi +# +# 1. Install Pixi +# curl -fsSL https://pixi.sh/install.sh | bash +# +# 2. Restart shell / terminal / console for Pixi to be recognized +# +# 3. Create Pixi environment +# pixi init +# +# 4. Setup Pixi environment +# cd +# git clone git@github.com:gafusion/gacode.git +# pixi add "python~=3.12,<3.13" +# pixi add git +# pixi add mpich +# pixi add gfortran +# pixi add openblas +# pixi add liblapack +# pixi add fftw +# pixi add hdf5 +# pixi add netcdf-fortran +# +# 5. Enter and configure Pixi environment (must be repeated every entry) +# pixi shell +# export GACODE_ROOT=${PIXI_PROJECT_ROOT}/gacode +# export GACODE_PLATFORM=CONDA +# source ${GACODE_ROOT}/shared/bin/gacode_setup +# +# 6. Build GACODE +# cd gacode +# make +# cd .. +# +# 7. Run regression tests +# neo -r +# tglf -r +# cgyro -r -n 4 -nomp 2 +# tgyro -r -n 4 +# +#--------------------------------------------------- + +MAKE = make +PREFIX = ${CONDA_PREFIX} +NETCDF_PATH=${PREFIX} +MF90 = mpif90 + +# Compilers and flags + +FC = ${MF90} -std=f2008 -fall-intrinsics -I$(GACODE_ROOT)/modules -J$(GACODE_ROOT)/modules -g -I${PREFIX}/include +F77 = ${MF90} -g + +FMATH = -fdefault-real-8 -fdefault-double-8 +FOPT = -O3 -m64 -fallow-argument-mismatch +FDEBUG = -Wall -fcheck=all -fbacktrace -fbounds-check -O0 -Wextra -finit-real=nan -Wunderflow -ffpe-trap=invalid,zero,overflow +FBOUND = -Wall -fbounds-check +FOMP = -fopenmp + +# System math libraries + +LMATH = -L${PREFIX}/lib -lfftw3 -llapack -lblas + +NETCDF = -L${PREFIX}/lib -lnetcdff -lnetcdf +NETCDF_INC =${PREFIX}/include + +# Archive + +ARCH = ar cr + From 42145515b10260a07aca26b5a367da641028c158 Mon Sep 17 00:00:00 2001 From: aaronkho Date: Fri, 25 Oct 2024 13:53:40 -0400 Subject: [PATCH 2/5] Added environment and exec files for pixi --- platform/env/env.PIXI | 2 ++ platform/exec/exec.PIXI | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 platform/env/env.PIXI create mode 100755 platform/exec/exec.PIXI diff --git a/platform/env/env.PIXI b/platform/env/env.PIXI new file mode 100644 index 000000000..a424ba80b --- /dev/null +++ b/platform/env/env.PIXI @@ -0,0 +1,2 @@ +# The PIXI platform assumes that you have installed pixi and run the +# appropriate commands to have the virtual environment set up well. diff --git a/platform/exec/exec.PIXI b/platform/exec/exec.PIXI new file mode 100755 index 000000000..d0010ad46 --- /dev/null +++ b/platform/exec/exec.PIXI @@ -0,0 +1,19 @@ +#!/bin/sh +# GACODE Parallel execution script (PIXI) +# +# NOTES: +# Used mpich2-1.0.1, so use mpirun rather than mpiexec + +simdir=${1} +nmpi=${2} +exec=${3} +nomp=${4} +numa=${5} +mpinuma=${6} + +echo $simdir + +cd $simdir + +mpirun -env OMP_NUM_THREADS $nomp -np $nmpi $exec + From 8d4cc83223ba9f0258c87a491c17ba03265d2017 Mon Sep 17 00:00:00 2001 From: aaronkho Date: Tue, 29 Oct 2024 13:30:19 -0400 Subject: [PATCH 3/5] Fix instructions to include OpenBLAS with OpenMP --- platform/build/make.inc.PIXI | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platform/build/make.inc.PIXI b/platform/build/make.inc.PIXI index 97750c263..188bbb352 100644 --- a/platform/build/make.inc.PIXI +++ b/platform/build/make.inc.PIXI @@ -17,6 +17,7 @@ # pixi add mpich # pixi add gfortran # pixi add openblas +# pixi add "libopenblas=*=*openmp*" # pixi add liblapack # pixi add fftw # pixi add hdf5 @@ -25,7 +26,7 @@ # 5. Enter and configure Pixi environment (must be repeated every entry) # pixi shell # export GACODE_ROOT=${PIXI_PROJECT_ROOT}/gacode -# export GACODE_PLATFORM=CONDA +# export GACODE_PLATFORM=PIXI # source ${GACODE_ROOT}/shared/bin/gacode_setup # # 6. Build GACODE From 94b8ed1137aa35ef148d11fb81b551498359f1a6 Mon Sep 17 00:00:00 2001 From: aaronkho Date: Thu, 14 Nov 2024 21:30:11 -0500 Subject: [PATCH 4/5] Split PIXI build into mpich and openmpi versions --- .../{make.inc.PIXI => make.inc.PIXI_MPICH} | 16 ++-- platform/build/make.inc.PIXI_OPENMP | 73 +++++++++++++++++++ platform/env/{env.PIXI => env.PIXI_MPICH} | 0 platform/env/env.PIXI_OPENMP | 2 + platform/exec/{exec.PIXI => exec.PIXI_MPICH} | 0 platform/exec/exec.PIXI_OPENMP | 19 +++++ 6 files changed, 103 insertions(+), 7 deletions(-) rename platform/build/{make.inc.PIXI => make.inc.PIXI_MPICH} (86%) create mode 100644 platform/build/make.inc.PIXI_OPENMP rename platform/env/{env.PIXI => env.PIXI_MPICH} (100%) create mode 100644 platform/env/env.PIXI_OPENMP rename platform/exec/{exec.PIXI => exec.PIXI_MPICH} (100%) create mode 100755 platform/exec/exec.PIXI_OPENMP diff --git a/platform/build/make.inc.PIXI b/platform/build/make.inc.PIXI_MPICH similarity index 86% rename from platform/build/make.inc.PIXI rename to platform/build/make.inc.PIXI_MPICH index 188bbb352..e16106e2c 100644 --- a/platform/build/make.inc.PIXI +++ b/platform/build/make.inc.PIXI_MPICH @@ -26,7 +26,7 @@ # 5. Enter and configure Pixi environment (must be repeated every entry) # pixi shell # export GACODE_ROOT=${PIXI_PROJECT_ROOT}/gacode -# export GACODE_PLATFORM=PIXI +# export GACODE_PLATFORM=PIXI_MPICH # source ${GACODE_ROOT}/shared/bin/gacode_setup # # 6. Build GACODE @@ -42,10 +42,10 @@ # #--------------------------------------------------- -MAKE = make -PREFIX = ${CONDA_PREFIX} -NETCDF_PATH=${PREFIX} -MF90 = mpif90 +MAKE = make +PREFIX = ${CONDA_PREFIX} +NETCDF_PATH = ${PREFIX} +MF90 = mpif90 # Compilers and flags @@ -62,8 +62,10 @@ FOMP = -fopenmp LMATH = -L${PREFIX}/lib -lfftw3 -llapack -lblas -NETCDF = -L${PREFIX}/lib -lnetcdff -lnetcdf -NETCDF_INC =${PREFIX}/include +# Optional netCDF libraries + +NETCDF = -L${PREFIX}/lib -lnetcdff -lnetcdf +NETCDF_INC = ${PREFIX}/include # Archive diff --git a/platform/build/make.inc.PIXI_OPENMP b/platform/build/make.inc.PIXI_OPENMP new file mode 100644 index 000000000..093574949 --- /dev/null +++ b/platform/build/make.inc.PIXI_OPENMP @@ -0,0 +1,73 @@ +#--------------------------------------------------- +# OSX, Linux mpich using pixi +# +# 1. Install Pixi +# curl -fsSL https://pixi.sh/install.sh | bash +# +# 2. Restart shell / terminal / console for Pixi to be recognized +# +# 3. Create Pixi environment +# pixi init +# +# 4. Setup Pixi environment +# cd +# git clone git@github.com:gafusion/gacode.git +# pixi add "python~=3.12,<3.13" +# pixi add git +# pixi add openmpi-mpifort +# pixi add gfortran +# pixi add openblas +# pixi add "libopenblas=*=*openmp*" +# pixi add liblapack +# pixi add fftw +# pixi add hdf5 +# pixi add netcdf-fortran +# +# 5. Enter and configure Pixi environment (must be repeated every entry) +# pixi shell +# export GACODE_ROOT=${PIXI_PROJECT_ROOT}/gacode +# export GACODE_PLATFORM=PIXI_OPENMPI +# source ${GACODE_ROOT}/shared/bin/gacode_setup +# +# 6. Build GACODE +# cd gacode +# make +# cd .. +# +# 7. Run regression tests +# neo -r +# tglf -r +# cgyro -r -n 4 -nomp 2 +# tgyro -r -n 4 +# +#--------------------------------------------------- + +MAKE = make +PREFIX = ${CONDA_PREFIX} +NETCDF_PATH = ${PREFIX} +MF90 = mpif90 + +# Compilers and flags + +FC = ${MF90} -std=f2008 -fall-intrinsics -I$(GACODE_ROOT)/modules -J$(GACODE_ROOT)/modules -g -I${PREFIX}/include +F77 = ${MF90} -g + +FMATH = -fdefault-real-8 -fdefault-double-8 +FOPT = -O3 -m64 -fallow-argument-mismatch +FDEBUG = -Wall -fcheck=all -fbacktrace -fbounds-check -O0 -Wextra -finit-real=nan -Wunderflow -ffpe-trap=invalid,zero,overflow +FBOUND = -Wall -fbounds-check +FOMP = -fopenmp + +# System math libraries + +LMATH = -L${PREFIX}/lib -lfftw3 -llapack -lblas + +# Optional netCDF libraries + +NETCDF = -L${PREFIX}/lib -lnetcdff -lnetcdf +NETCDF_INC = ${PREFIX}/include + +# Archive + +ARCH = ar cr + diff --git a/platform/env/env.PIXI b/platform/env/env.PIXI_MPICH similarity index 100% rename from platform/env/env.PIXI rename to platform/env/env.PIXI_MPICH diff --git a/platform/env/env.PIXI_OPENMP b/platform/env/env.PIXI_OPENMP new file mode 100644 index 000000000..a424ba80b --- /dev/null +++ b/platform/env/env.PIXI_OPENMP @@ -0,0 +1,2 @@ +# The PIXI platform assumes that you have installed pixi and run the +# appropriate commands to have the virtual environment set up well. diff --git a/platform/exec/exec.PIXI b/platform/exec/exec.PIXI_MPICH similarity index 100% rename from platform/exec/exec.PIXI rename to platform/exec/exec.PIXI_MPICH diff --git a/platform/exec/exec.PIXI_OPENMP b/platform/exec/exec.PIXI_OPENMP new file mode 100755 index 000000000..ce2550ed3 --- /dev/null +++ b/platform/exec/exec.PIXI_OPENMP @@ -0,0 +1,19 @@ +#!/bin/sh +# GACODE Parallel execution script (PIXI) +# +# NOTES: +# Used mpich2-1.0.1, so use mpirun rather than mpiexec + +simdir=${1} +nmpi=${2} +exec=${3} +nomp=${4} +numa=${5} +mpinuma=${6} + +echo $simdir + +cd $simdir + +mpirun -x OMP_NUM_THREADS=$nomp -np $nmpi $exec + From 883d19760c51f8b6eb2007ade28b7e9866c863c6 Mon Sep 17 00:00:00 2001 From: aaronkho Date: Thu, 14 Nov 2024 23:35:36 -0500 Subject: [PATCH 5/5] Enable oversubscribe in PIXI_OPENMP build --- platform/exec/exec.PIXI_OPENMP | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platform/exec/exec.PIXI_OPENMP b/platform/exec/exec.PIXI_OPENMP index ce2550ed3..f4f1798b1 100755 --- a/platform/exec/exec.PIXI_OPENMP +++ b/platform/exec/exec.PIXI_OPENMP @@ -1,8 +1,8 @@ #!/bin/sh -# GACODE Parallel execution script (PIXI) +# GACODE Parallel execution script (PIXI_OPENMP) # # NOTES: -# Used mpich2-1.0.1, so use mpirun rather than mpiexec +# Used openmpi-5.0.1 simdir=${1} nmpi=${2} @@ -15,5 +15,5 @@ echo $simdir cd $simdir -mpirun -x OMP_NUM_THREADS=$nomp -np $nmpi $exec +mpirun -x OMP_NUM_THREADS=$nomp -np $nmpi --oversubscribe $exec