diff --git a/platform/build/make.inc.PPPL_FLUX b/platform/build/make.inc.PPPL_FLUX new file mode 100644 index 000000000..bbdfc9b16 --- /dev/null +++ b/platform/build/make.inc.PPPL_FLUX @@ -0,0 +1,37 @@ +#--------------------------------------------------- +# PPPL Systems (FLUX) +# with Intel Compiler and OpenMPI +#--------------------------------------------------- +# L. Peterson (jpeterso@pppl.gov) +# D. Mikkelsen (mikk@pppl.gov) +# N. Ferraro (nferraro@pppl.gov) +# B.A. Grierson (bgriers@pppl.gov) +# J Parisi -- jparisi@pppl.gov + +IDENTITY="PPPL FLUX cluster" +CORES_PER_NODE=1 +NUMAS_PER_NODE=1 + +# Compilers (intel) and flags +FC = mpif90 -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/modules -I${MKLROOT}/include/fftw -I$(NETCDF_FORTRAN_HOME)/include +F77 = ${FC} + +FOMP = -qopenmp +FMATH = -real-size 64 +FOPT = -O2 +FDEBUG = -check all -traceback +F2PY = f2py + +# System math libraries +LMATH = -mkl + +# NetCDF +NETCDF = -L${NETCDF_FORTRAN_HOME}/lib -lnetcdff -L${NETCDF_C_HOME}/lib -lnetcdf +NETCDF_INC =-I${NETCDF_FORTRAN_HOME}/include -I${NETCDF_C_HOME}/include + +# Mapping +%.o : %.mod + +# Archive +ARCH = ar cr + diff --git a/platform/build/make.inc.PPPL_PORTAL b/platform/build/make.inc.PPPL_PORTAL new file mode 100755 index 000000000..ff964c799 --- /dev/null +++ b/platform/build/make.inc.PPPL_PORTAL @@ -0,0 +1,35 @@ +#--------------------------------------------------- +# PPPL Portal Systems (ellis, dawson, kestrel, kite and kruskal) +# with gcc compiler and OpenMPI, updated as of April 25 2024. +#--------------------------------------------------- +# S. Smith (smithsp@fusion.gat.com) +# J. Parisi (jparisi@pppl.gov) + +IDENTITY="PPPL Portal cluster" +CORES_PER_NODE=1 +NUMAS_PER_NODE=1 + +# Compilers and flags +FC = mpif90 -std=f2008 -fall-intrinsics -I$(GACODE_ROOT)/modules -J$(GACODE_ROOT)/modules -fPIC +F77 = mpif77 -std=legacy -I$(GACODE_ROOT)/modules +CC = gcc + +FMATH = -fdefault-real-8 -fdefault-double-8 +FOPT = -Ofast +FDEBUG = -Wall -W -fcheck=mem,bounds -g -fbacktrace -ffpe-trap=invalid,zero,overflow +F2PY = f2py +FOMP = -fopenmp + +# System math libraries +LMATH = -L/usr/lib64 -lopenblas $(FFTW_HOME)/lib/libfftw3.a $(FFTW_HOME)/lib/libfftw3_omp.a +FFTW_INC = $(FFTW_HOME)/include + +# NetCDF +NETCDF = -L/usr/pppl/gcc/11.2-pkgs/netcdf-fortran-4.5.4/lib -lnetcdff -L/usr/lib64/hdf -L/usr/pppl/gcc/11.2-pkgs/netcdf-c-4.8.1/lib -lnetcdf -lnetcdf -ldl -lm +NETCDF_INC =${NETCDF_FORTRAN_HOME}/include -I${NETCDF_C_HOME}/include + +# Mapping +%.o : %.mod + +# Archive +ARCH = ar cr diff --git a/platform/env/env.PPPL_FLUX b/platform/env/env.PPPL_FLUX new file mode 100644 index 000000000..bd2715d97 --- /dev/null +++ b/platform/env/env.PPPL_FLUX @@ -0,0 +1,11 @@ +#!/bin/bash +#module purge +module load intel@2023.2.0 +module load openmpi-4.1.5/intel-2023.2.0/netcdf-c/4.9.2-3wxlq +module load openmpi-4.1.5/intel-2023.2.0/netcdf-cxx4/4.3.1-frhsv +module load openmpi-4.1.5/intel-2023.2.0/netcdf-fortran/4.6.0-hz3b2 +module load openmpi-4.1.5/intel-2023.2.0/fftw/3.3.10-zp3rj +module load openmpi-4.1.5/intel-2023.2.0/hdf5/1.14.1-2-x7cbk +module load git +export NETCDF_FORTRAN_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/netcdf-fortran-4.6.0-hz3b2mq6ld2u2u6ktdodbx72k37opcdf +export FFTW_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/fftw-3.3.10-zp3rj3oyvya5ntql27775l5gsfyou6ga diff --git a/platform/env/env.PPPL_PORTAL b/platform/env/env.PPPL_PORTAL new file mode 100644 index 000000000..8f9ac678f --- /dev/null +++ b/platform/env/env.PPPL_PORTAL @@ -0,0 +1,7 @@ +#!/bin/bash + +module load gcc/11.2.0 +module load openmpi/4.1.2 +module load fftw/3.3.10 +module load netcdf-c/4.8.1 +module load netcdf-fortran/4.5.4 diff --git a/platform/exec/exec.PPPL_FLUX b/platform/exec/exec.PPPL_FLUX new file mode 100755 index 000000000..d9177d4fe --- /dev/null +++ b/platform/exec/exec.PPPL_FLUX @@ -0,0 +1,16 @@ +#!/bin/sh +# GACODE Parallel execution script (PPPL FLUX) + +simdir=${1} +nmpi=${2} +exec=${3} +nomp=${4} +numa=${5} +mpinuma=${6} + +cd $simdir + +# Had to remove --oversubscribe for NEO to run for Tom Osborne +echo "export OMP_NUM_THREADS=$nomp ; mpirun -np --oversubscribe $nmpi $exec " +export OMP_NUM_THREADS=$nomp ; mpirun --oversubscribe -np $nmpi $exec +exit diff --git a/platform/exec/exec.PPPL_PORTAL b/platform/exec/exec.PPPL_PORTAL new file mode 100755 index 000000000..e6f4221a6 --- /dev/null +++ b/platform/exec/exec.PPPL_PORTAL @@ -0,0 +1,16 @@ +#!/bin/sh +# GACODE Parallel execution script (PPPL_PORTAL) + +simdir=${1} +nmpi=${2} +exec=${3} +nomp=${4} +numa=${5} +mpinuma=${6} + +cd $simdir + +# Had to remove --oversubscribe for NEO to run for Tom Osborne +echo "export OMP_NUM_THREADS=$nomp ; mpirun -np --oversubscribe $nmpi $exec " +export OMP_NUM_THREADS=$nomp ; mpirun --oversubscribe -np $nmpi $exec +exit diff --git a/platform/modulefiles/gacode/PPPL_FLUX b/platform/modulefiles/gacode/PPPL_FLUX new file mode 100644 index 000000000..43cce7c1b --- /dev/null +++ b/platform/modulefiles/gacode/PPPL_FLUX @@ -0,0 +1,15 @@ +#%Module +# +# Module file for setting the GACODE machine-dependent environmental variables +# + +module purge +module load intel@2023.2.0 +module load openmpi-4.1.5/intel-2023.2.0/netcdf-c/4.9.2-3wxlq +module load openmpi-4.1.5/intel-2023.2.0/netcdf-cxx4/4.3.1-frhsv +module load openmpi-4.1.5/intel-2023.2.0/netcdf-fortran/4.6.0-hz3b2 +module load openmpi-4.1.5/intel-2023.2.0/fftw/3.3.10-zp3rj +module load openmpi-4.1.5/intel-2023.2.0/hdf5/1.14.1-2-x7cbk +module load git +export NETCDF_FORTRAN_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/netcdf-fortran-4.6.0-hz3b2mq6ld2u2u6ktdodbx72k37opcdf +export FFTW_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/fftw-3.3.10-zp3rj3oyvya5ntql27775l5gsfyou6ga diff --git a/platform/modulefiles/gacode/PPPL_PORTAL b/platform/modulefiles/gacode/PPPL_PORTAL new file mode 100755 index 000000000..18b838f70 --- /dev/null +++ b/platform/modulefiles/gacode/PPPL_PORTAL @@ -0,0 +1,23 @@ +#%Module +# +# Module file for setting the GACODE machine-dependent environmental variables +# +if { [ module-info mode load ] } { + +module load mod_ntcc +module load ntcc/gcc +module load lapack +module load fftw +module load fann + +} + +# module load gacode_base + +if { [ module-info mode remove ] } { +module load fann +module load fftw +module load lapack +module load ntcc/gcc +module load mod_ntcc +} diff --git a/platform/qsub/qsub.PPPL_FLUX b/platform/qsub/qsub.PPPL_FLUX new file mode 100644 index 000000000..d1d4521e4 --- /dev/null +++ b/platform/qsub/qsub.PPPL_FLUX @@ -0,0 +1,16 @@ + +bfile=$SIMDIR/batch.src +echo "#!/bin/bash " > $bfile +echo "#SBATCH -J $LOCDIR" >> $bfile +echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile +echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile +echo "#SBATCH -t $WALLTIME" >> $bfile +echo "#SBATCH -n $cores_used" >> $bfile +echo "#SBATCH --mem=2GB" >> $bfile +if [ "$QUEUE" = "null_queue" ] +then + echo "#SBATCH -p general" >> $bfile +else + echo "#SBATCH -p $QUEUE" >> $bfile +fi +echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile diff --git a/platform/qsub/qsub.PPPL_PORTAL b/platform/qsub/qsub.PPPL_PORTAL new file mode 100755 index 000000000..a47805fc5 --- /dev/null +++ b/platform/qsub/qsub.PPPL_PORTAL @@ -0,0 +1,25 @@ + +bfile=$SIMDIR/batch.src +echo "#!/bin/bash " > $bfile +echo "#SBATCH -J $LOCDIR" >> $bfile +echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile +echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile +echo "#SBATCH -t $WALLTIME" >> $bfile +echo "#SBATCH -n $cores_used" >> $bfile + +if [ -n $MEMPERNODE ] +then + echo "#SBATCH --mem $MEMNODE" >> $bfile +elif [ -n $MEMPERCPU ] + echo "#SBATCH --mem-per-cpu $MEMPERCPU" >> $bfile +else + echo "#SBATCH --mem-per-cpu 4GB" >> $bfile +fi + +if [ "$QUEUE" = "null_queue" ] +then + echo "#SBATCH -p general" >> $bfile +else + echo "#SBATCH -p $QUEUE" >> $bfile +fi +echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile