Updating PPPL platform files for Portal and Flux clusters, with GACOD…

…E_PLATFORM = PPPL_PORTAL and PPPL_FLUX

platform/build/make.inc.PPPL_FLUX

@@ -0,0 +1,37 @@
+#---------------------------------------------------
+# PPPL Systems (FLUX) 
+#               with Intel Compiler and OpenMPI
+#---------------------------------------------------
+# L. Peterson ([email protected])
+# D. Mikkelsen ([email protected])
+# N. Ferraro ([email protected])
+# B.A. Grierson ([email protected])
+# J Parisi -- [email protected]
+
+IDENTITY="PPPL FLUX cluster"
+CORES_PER_NODE=1
+NUMAS_PER_NODE=1
+
+# Compilers (intel) and flags 
+FC     = mpif90 -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/modules -I${MKLROOT}/include/fftw -I$(NETCDF_FORTRAN_HOME)/include
+F77    = ${FC}
+
+FOMP   = -qopenmp
+FMATH  = -real-size 64
+FOPT   = -O2
+FDEBUG = -check all -traceback
+F2PY   = f2py
+
+# System math libraries
+LMATH = -mkl
+
+# NetCDF
+NETCDF = -L${NETCDF_FORTRAN_HOME}/lib -lnetcdff -L${NETCDF_C_HOME}/lib -lnetcdf
+NETCDF_INC =-I${NETCDF_FORTRAN_HOME}/include -I${NETCDF_C_HOME}/include
+
+# Mapping
+%.o : %.mod
+
+# Archive 
+ARCH = ar cr
+

platform/build/make.inc.PPPL_PORTAL

@@ -0,0 +1,35 @@
+#---------------------------------------------------
+# PPPL Portal Systems (ellis, dawson, kestrel, kite and kruskal) 
+#               with gcc compiler and OpenMPI, updated as of April 25 2024.
+#---------------------------------------------------
+# S. Smith ([email protected])
+# J. Parisi ([email protected])
+
+IDENTITY="PPPL Portal cluster"
+CORES_PER_NODE=1
+NUMAS_PER_NODE=1
+
+# Compilers and flags 
+FC  = mpif90 -std=f2008 -fall-intrinsics -I$(GACODE_ROOT)/modules -J$(GACODE_ROOT)/modules -fPIC
+F77 = mpif77 -std=legacy -I$(GACODE_ROOT)/modules
+CC  = gcc
+
+FMATH  = -fdefault-real-8 -fdefault-double-8 
+FOPT   = -Ofast
+FDEBUG = -Wall -W -fcheck=mem,bounds -g -fbacktrace -ffpe-trap=invalid,zero,overflow
+F2PY   = f2py
+FOMP   = -fopenmp
+
+# System math libraries
+LMATH = -L/usr/lib64 -lopenblas $(FFTW_HOME)/lib/libfftw3.a $(FFTW_HOME)/lib/libfftw3_omp.a
+FFTW_INC = $(FFTW_HOME)/include
+
+# NetCDF
+NETCDF = -L/usr/pppl/gcc/11.2-pkgs/netcdf-fortran-4.5.4/lib -lnetcdff -L/usr/lib64/hdf -L/usr/pppl/gcc/11.2-pkgs/netcdf-c-4.8.1/lib -lnetcdf -lnetcdf -ldl -lm
+NETCDF_INC =${NETCDF_FORTRAN_HOME}/include -I${NETCDF_C_HOME}/include
+
+# Mapping
+%.o : %.mod
+
+# Archive 
+ARCH = ar cr

platform/env/env.PPPL_FLUX

@@ -0,0 +1,11 @@
+#!/bin/bash
+#module purge
+module load [email protected]
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-c/4.9.2-3wxlq
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-cxx4/4.3.1-frhsv
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-fortran/4.6.0-hz3b2
+module load openmpi-4.1.5/intel-2023.2.0/fftw/3.3.10-zp3rj
+module load openmpi-4.1.5/intel-2023.2.0/hdf5/1.14.1-2-x7cbk
+module load git
+export NETCDF_FORTRAN_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/netcdf-fortran-4.6.0-hz3b2mq6ld2u2u6ktdodbx72k37opcdf
+export FFTW_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/fftw-3.3.10-zp3rj3oyvya5ntql27775l5gsfyou6ga

platform/env/env.PPPL_PORTAL

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+module load gcc/11.2.0 
+module load openmpi/4.1.2
+module load fftw/3.3.10
+module load netcdf-c/4.8.1
+module load netcdf-fortran/4.5.4

platform/exec/exec.PPPL_FLUX

@@ -0,0 +1,16 @@
+#!/bin/sh
+# GACODE Parallel execution script (PPPL FLUX)
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+
+cd $simdir
+
+# Had to remove --oversubscribe for NEO to run for Tom Osborne
+echo "export OMP_NUM_THREADS=$nomp ; mpirun -np --oversubscribe $nmpi $exec "
+export OMP_NUM_THREADS=$nomp ; mpirun --oversubscribe -np $nmpi  $exec
+exit

platform/exec/exec.PPPL_PORTAL

@@ -0,0 +1,16 @@
+#!/bin/sh
+# GACODE Parallel execution script (PPPL_PORTAL)
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+
+cd $simdir
+
+# Had to remove --oversubscribe for NEO to run for Tom Osborne
+echo "export OMP_NUM_THREADS=$nomp ; mpirun -np --oversubscribe $nmpi $exec "
+export OMP_NUM_THREADS=$nomp ; mpirun --oversubscribe -np $nmpi  $exec
+exit

platform/modulefiles/gacode/PPPL_FLUX

@@ -0,0 +1,15 @@
+#%Module
+#
+# Module file for setting the GACODE machine-dependent environmental variables
+#
+
+module purge
+module load [email protected]
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-c/4.9.2-3wxlq
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-cxx4/4.3.1-frhsv
+module load openmpi-4.1.5/intel-2023.2.0/netcdf-fortran/4.6.0-hz3b2
+module load openmpi-4.1.5/intel-2023.2.0/fftw/3.3.10-zp3rj
+module load openmpi-4.1.5/intel-2023.2.0/hdf5/1.14.1-2-x7cbk
+module load git
+export NETCDF_FORTRAN_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/netcdf-fortran-4.6.0-hz3b2mq6ld2u2u6ktdodbx72k37opcdf
+export FFTW_HOME=/opt/pppl/spack-pkgs/linux-rocky9-zen3/intel-2023.2.0/fftw-3.3.10-zp3rj3oyvya5ntql27775l5gsfyou6ga

platform/modulefiles/gacode/PPPL_PORTAL

@@ -0,0 +1,23 @@
+#%Module
+#
+# Module file for setting the GACODE machine-dependent environmental variables
+#
+if { [ module-info mode load ] } {
+
+module load mod_ntcc
+module load ntcc/gcc
+module load lapack
+module load fftw
+module load fann
+
+}
+
+# module load gacode_base
+
+if { [ module-info mode remove ] } {
+module load fann
+module load fftw
+module load lapack
+module load ntcc/gcc
+module load mod_ntcc
+}

platform/qsub/qsub.PPPL_FLUX

@@ -0,0 +1,16 @@
+
+bfile=$SIMDIR/batch.src
+echo "#!/bin/bash " > $bfile
+echo "#SBATCH -J $LOCDIR" >> $bfile
+echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile
+echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile
+echo "#SBATCH -t $WALLTIME" >> $bfile
+echo "#SBATCH -n $cores_used" >> $bfile
+echo "#SBATCH --mem=2GB" >> $bfile
+if [ "$QUEUE" = "null_queue" ]
+then
+  echo "#SBATCH -p general" >> $bfile
+else
+  echo "#SBATCH -p $QUEUE" >> $bfile
+fi
+echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile

platform/qsub/qsub.PPPL_PORTAL

@@ -0,0 +1,25 @@
+
+bfile=$SIMDIR/batch.src
+echo "#!/bin/bash " > $bfile
+echo "#SBATCH -J $LOCDIR" >> $bfile
+echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile
+echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile
+echo "#SBATCH -t $WALLTIME" >> $bfile
+echo "#SBATCH -n $cores_used" >> $bfile
+
+if [ -n $MEMPERNODE ] 
+then
+  echo "#SBATCH --mem $MEMNODE" >> $bfile
+elif [ -n $MEMPERCPU ]
+  echo "#SBATCH --mem-per-cpu $MEMPERCPU" >> $bfile
+else
+  echo "#SBATCH --mem-per-cpu 4GB" >> $bfile
+fi
+
+if [ "$QUEUE" = "null_queue" ]
+then
+  echo "#SBATCH -p general" >> $bfile
+else
+  echo "#SBATCH -p $QUEUE" >> $bfile
+fi
+echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile