Add compile and platform yamls to compile model with FRE/2025.01

yamls/NWA12/CEFI_NWA12_cobalt.yaml

@@ -1,43 +1,68 @@
-# POSTPROCESSING GUIDE
+# USAGE
+#
+# The following yaml is designed to compile and postprocess the ocean_monthly component of the CEFI_NWA12_COBALT_V1 experiment
+# Full Documentation is available here: https://noaa-gfdl.github.io/fre-cli/index.html
+# 
+# 0.) Set up your environment by running "module use -a /ncrc/home2/fms/local/modulefiles" followed by "module load fre/2025.01"
+#
+# For modularity and compatibility with with cylc workflow engine, fre/2025.01 splits up the compile and post processcing process 
+# into several sub steps.
 #
-# The following yaml is designed to postprocess the ocean_monthly component of the 2024_06 version of the CEFI_NWA12_COBALT_V1 experiment
-# if postprocessing data from a different version of this experiment, be sure to change the FRE_STEM variable before postprocessing
+# COMPILE GUIDE - BAREMETAL RUN
+# 
+# 1.) Create script that will be used to checkout model components and OPTIONALLY run it with the --execute flag. You can manually run the
+# checkout script after this step as well
+# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute
+# 
+# 2.) Create Makefile
+# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod
 #
-# Quick start instructions and source code can be found here: https://github.com/NOAA-GFDL/fre-cli/tree/main/fre/pp
+# 3.) Create script that will compile the model, and OPTIONALLY run it with the --execute flag. You can manually run the 
+# compile script after this step as well
+# fre make create-compile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute
 #
-# 0.) Load module on GFDL Analysis: module load fre/canopy
+# COMPILE GUIDE - CONTAINERS
 #
-# For modularity and compatibility with with cylc workflow engine, fre/canopy splits up the post processcing process into several sub steps:
+# 1.) Create checkout script with the "no parallel checkouts option" - this is required for container builds
+# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod -npc
+# 
+# 2.) Create Makefile AND a Docker file. You can then build the container by manually running the createContainer.sh
+# script, or by OPTIONALLY including the --execute file when creating the container
+# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod
+# fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute
 #
+# POSTPROCESSING GUIDE
+#
+# 0.) Load module on GFDL Analysis: module load fre/2025.01
+# 
 # 1.) Checkout the git repo containing postprocessing scripts and related files with the folowing command:
-# fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod
+# fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
 #
 # 2.) Combine your main yaml and experiment yamls into a single yaml, then set up the cylc-src dir with the configure-yaml command:
-# fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y CEFI_NWA12_cobalt.yaml
-# fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y combined-CEFI_NWA12_COBALT_V1.yaml
+# fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml
+# fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml --use pp
 #
-#     2a.) Fre/canopy does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator:
-#     mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/pp
+#     2a.) fre/2025.01 does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator:
+#     mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/pp
 #
 #     2b.) (OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors
-#     tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel22__prod/history-manifest
+#     tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel23__prod/history-manifest
 #
 # 3.) Validate that all configuration files are correct
-# fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod
+# fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
 #
 # 4.) Create the cylc-run directory containing the final version of configuration files, scripts, and output directories
-# fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod
+# fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
 #
 # 5.) Run post processing
-# fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod
+# fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
 #
 # 6.) To monitor the status of each post processing step, run the following command:
-# fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod
-
+# fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
 
 
 fre_properties: 
-    - &FRE_STEM       "fre/cefi/NWA/2024_06"
+    - &FRE_STEM       "fre/cefi/NWA/ShortCanopyTest"
 
     - &PP_START_YEAR  "1993"
     - &PP_END_YEAR    "2019"
@@ -47,7 +72,17 @@ fre_properties:
 
     - &GRID_SPEC      "/archive/acr/mom6_input/nwa12/nwa12_grid_75z.tar"
 
-
+    # Compile properties
+    - &FMSincludes    "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+    - &F2003_FLAGS    " -DINTERNAL_FILE_NML -g "
+    - &momIncludes    "-Imom6/src/MOM6/pkg/CVMix-src/include"
+    - &sisincludes    "-Imom6/src/MOM6/src/framework"
+    - &MOM6_GIT_FIX   "main"
+    - &FMS_GIT_TAG    "2024.03"
+    - &CPL_GIT_TAG    "2024.03"
+    - &INTEL          "intel-classic"
+
+
 shared: 
     directories: &shared_directories
         history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
@@ -77,6 +112,21 @@ shared:
         compileYaml: &compile_yaml
         platformYaml:
 
+#################################################################################################
+# COMPILE                                                                                       #
+#################################################################################################
+
+## The location for yamls
+build:
+  platformYaml: platforms.yaml
+  compileYaml: compile.yaml
+## User defined properties for building
+#release: "2024.01"
+
+#################################################################################################
+# POST-PROCESSING                                                                               #
+#################################################################################################
+
 experiments: 
     - name: "MOM6_SIS2_GENERIC_4P_compile_symm"
       pp:

yamls/NWA12/README.md

@@ -0,0 +1,119 @@
+# FRE yamls
+
+The latest version of the FMS Runtime Environment uses yamls instead of xmls to execute the FRE workflow. This folder contains experiment, compile, and platform yamls that can be used with the latest version of `fre` to compile and post process the `CEFI_NWA12_Cobalt_V1` experiment. The compiled experiment can then be run in conjuction with `fre/bronx-23` and the existing xmls in the `xml/NWA12` directory. 
+
+The following sections assume that you are compiling and running experiments on Gaea C5, and then conducting postprocessing on GFDL's PPAN system. The `platforms.yaml` does support compilation on C6, but `bronx-23` does not support containerized runs on C6 - baremetal runs, however, are supported on C6.
+
+Full documentation for `Fre/2025.01` is available [here](https://noaa-gfdl.github.io/fre-cli/usage.html)
+
+## Compile the Experiment. 
+
+Begin by loading the appropriate modules on Gaea C5: 
+```
+module use -a /ncrc/home2/fms/local/modulefiles
+module load fre/2025.01
+```
+
+`fre/2025.01` supports both containerized compilation and bare-metal compilation, with very similar steps for both targets. 
+
+### Baremetal Compilation 
+
+Run the following command to create the script that will be used to checkout the model components, and optionally run the script by passing in the --execute flag.
+If you don't provide the --execute flag, be sure to run the script printed at the end of this command manually.
+```
+fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute
+```
+
+Create the Makefile for the experiment: 
+```
+fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod
+```
+
+Create the script that actually compiles the experiment, and optionally run the script by passing in the --execute flag
+```
+fre make create-compile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute
+```
+
+### Containerized Compilation
+
+Compiling the experiment into a container is a similar process. The only changes are 1.) passing in the -npc flag during the checkout step to prevent parallel checkouts, 2.) changing you platform to `hpcme.2023` and 3.) creating a Dockerfile and container creation script instead of a compilation script. Note that the Dockerfile will compile the model within the container as part of the build process.
+
+```
+fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod -npc
+fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod
+fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute
+```
+
+## Running the Experiment
+
+`fre/2025.01` does not currently support running experiments with yamls, so in order to run the compiled experiment in fre, you will have to use the existing xmls along with `fre/bronx`. Make sure the `FRE_STEM` set in the xml matches the `FRE_STEM` set in the experiment yaml.
+
+**NOTE**: Since capitilized names are not allowed in Dockerfiles, the current compile yaml names the compilation experiment `mom6_sis2_generic_4p_compile_symm_yaml` instead of `MOM6_SIS2_GENERIC_4P_compile_symm_yaml` like the xml. As a result, `fre` will print a warning that it cannot find your executable when you run `frerun`, and will eventually fail if you submit the job.
+
+To get around this, change the path set in the `executable` variable within the runscript printed by `frerun` to point to the exectable located in `${FRE_STEM}/mom6_sis2_generic_4p_compile_symm_yaml/exec` instead.
+
+### Baremetal Run
+
+Baremetal experiments can be run using the same run [instructions](https://github.com/NOAA-GFDL/CEFI-regional-MOM6/tree/main/xmls) available in the xmls directory. Be sure to load the modules described in that page before running `frerun`. 
+
+### Containerized Run
+Current xmls can be used for containerized runs, as well, so long as the following changes are made: 
+
+1.) Run `module load fre/bronx-23` and change the `FRE_VERSION` property of the xml to `bronx-23`.
+2.) Right below the `<experiment name="CEFI_NWA12_COBALT_V1" inherit="MOM6_SIS2_GENERIC_4P_compile_symm">` tag, add a tag with a path to your `.sif` singularity image file: 
+    ```
+    <container file="path/to/container/"/>
+    ```
+
+With these changes, you should be able to run the experiment by calling `frerun` with an extra `--container` flag
+```
+frerun -x CEFI_NWA12_cobalt.xml -p ncrc5.intel23 -t prod CEFI_NWA12_COBALT_V1 --container
+```
+
+
+## Postprocessing the Experiment
+Like `fre make`, postprocessing is split into several subcommands to improve modularity. On PPAN, load the appropriate module: 
+```
+module load fre/2025.01
+```
+
+Checkout the git repo containing postprocessing scripts and related files with the folowing command:
+```
+fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
+```
+
+Combine your main yaml and experiment yamls into a single yaml, then set up the cylc-src dir with the configure-yaml command:
+```
+fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml --use pp
+fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml
+```
+
+`fre/2025.01` does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator:
+```
+mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/pp
+```
+
+(OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors
+```
+tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel23__prod/history-manifest
+```
+
+Validate that all configuration files are correct
+```
+fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
+```
+
+Create the cylc-run directory containing the final version of configuration files, scripts, and output directories
+```
+fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
+```
+
+Run post processing
+```
+fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
+```
+
+To monitor the status of each post processing step, run the following command:
+```
+fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod
+```

yamls/NWA12/compile.yaml

@@ -0,0 +1,65 @@
+compile:
+  experiment: "mom6_sis2_generic_4p_compile_symm_yaml"
+  container_addlibs:
+  baremetal_linkerflags:
+  src:
+       - component: FMS
+         paths: [FMS]
+         repo: "https://github.com/NOAA-GFDL/FMS.git"
+         branch: *FMS_GIT_TAG
+         cppdefs: " -DINTERNAL_FILE_NML -g -Duse_libMPI -Duse_netCDF -Duse_yaml  -DMAXFIELDMETHODS_=600" #this uses deprecated IO
+         otherFlags: *FMSincludes
+
+       - component: mom6
+         requires: [FMS]
+         repo: "https://github.com/NOAA-GFDL/CEFI-regional-MOM6.git"
+         branch: *MOM6_GIT_FIX
+         paths: [mom6/src/MOM6/config_src/memory/dynamic_symmetric, 
+                 mom6/src/MOM6/config_src/drivers/FMS_cap,
+                 mom6/src/MOM6/src/*/,
+                 mom6/src/MOM6/src/*/*/,
+                 mom6/src/MOM6/config_src/external/ODA_hooks,
+                 mom6/src/MOM6/config_src/external/stochastic_physics,
+                 mom6/src/MOM6/config_src/external/drifters,
+                 mom6/src/MOM6/config_src/external/database_comms,
+                 mom6/src/ocean_BGC/generic_tracers,
+                 mom6/src/ocean_BGC/mocsy/src,
+                 mom6/src/MOM6/pkg/GSW-Fortran/modules,
+                 mom6/src/MOM6/pkg/GSW-Fortran/toolbox,
+                 mom6/src/MOM6/config_src/infra/FMS2]
+         #makeOverrides: 'OPENMP=""'
+         otherFlags: !join [*FMSincludes," ", *momIncludes]
+         cppdefs: "-DINTERNAL_FILE_NML -g -DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DUSE_FMS2_IO 
+           -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER  -DUSE_PRECISION=2 "
+
+       - component: sis2
+         requires: [FMS, mom6]
+         repo: "https://github.com/NOAA-GFDL/ice_param.git"
+         paths: [mom6/src/SIS2/config_src/dynamic_symmetric,
+                 mom6/src/SIS2/config_src/external/Icepack_interfaces,
+                 mom6/src/SIS2/src,
+                 mom6/src/icebergs/src,
+                 mom6/src/ice_param]
+         otherFlags: !join [*FMSincludes," ", *momIncludes," ", *sisincludes]
+         #makeOverrides: 'OPENMP=""'
+         cppdefs: -DINTERNAL_FILE_NML -g -DUSE_FMS2_IO 
+
+       - component: land_null
+         requires: [FMS]
+         repo: "https://github.com/NOAA-GFDL/land_null.git"
+         paths: [mom6/src/land_null]
+
+       - component: atmos_null
+         requires: [FMS]
+         repo: "https://github.com/NOAA-GFDL/atmos_null.git"
+         paths: [ mom6/src/atmos_null ]
+         otherFlags: !join [*FMSincludes," ", *momIncludes]
+         cppdefs: -DINTERNAL_FILE_NML -g
+
+       - component: coupler
+         repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+         requires: [FMS, mom6, sis2, land_null, atmos_null]
+         paths: [mom6/src/coupler/shared,
+                 mom6/src/coupler/full ]
+         otherFlags: !join [*FMSincludes," ", *momIncludes]
+         cppdefs: -DINTERNAL_FILE_NML -g -DUSE_FMS2_IO -D_USE_LEGACY_LAND_ -Duse_AM3_physics

yamls/NWA12/platforms.yaml

@@ -0,0 +1,25 @@
+platforms:
+   - name: ncrc5.intel23
+     compiler: intel
+     modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+     modules: [!join [*INTEL, /2023.2.0], fre/bronx-22, cray-hdf5, cray-netcdf]
+     mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc5/", *INTEL, ".mk"]
+     modelRoot: !join ["/gpfs/f5/cefi/scratch/${USER}", "/", *FRE_STEM ]
+
+   - name: ncrc6.intel23
+     compiler: intel
+     modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+     modules: [!join [*INTEL, /2023.2.0], fre/bronx-22, cray-hdf5, cray-netcdf]
+     mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc6/", *INTEL, ".mk"]
+     modelRoot: !join ["/gpfs/f6/ira-cefi/scratch/${USER}", "/", *FRE_STEM ]
+
+   - name: hpcme.2023
+     compiler: intel
+     RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
+     modelRoot: /apps
+     container: true
+     containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25"
+     containerBuild: "podman"
+     containerRun: "apptainer" 
+     # NOTE: c5 and c6 make templates are currently the same. Defaulting to c6 for container runs
+     mkTemplate: "/apps/mkmf/templates/hpcme-intel23.mk"