From 2f7b2ba7a87338437020b72effe6a2fd01d71bd2 Mon Sep 17 00:00:00 2001 From: Utheri Wagura Date: Wed, 8 Jan 2025 15:56:14 -0500 Subject: [PATCH 1/4] Add compile and platform yamls, modify experiment yaml to work with them --- yamls/NWA12/CEFI_NWA12_cobalt.yaml | 30 ++++++++++++-- yamls/NWA12/compile.yaml | 65 ++++++++++++++++++++++++++++++ yamls/NWA12/platforms.yaml | 49 ++++++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 yamls/NWA12/compile.yaml create mode 100644 yamls/NWA12/platforms.yaml diff --git a/yamls/NWA12/CEFI_NWA12_cobalt.yaml b/yamls/NWA12/CEFI_NWA12_cobalt.yaml index 4652af80f..80c5f2ceb 100644 --- a/yamls/NWA12/CEFI_NWA12_cobalt.yaml +++ b/yamls/NWA12/CEFI_NWA12_cobalt.yaml @@ -35,9 +35,8 @@ # fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod - fre_properties: - - &FRE_STEM "fre/cefi/NWA/2024_06" + - &FRE_STEM "fre/cefi/NWA/2025_01" - &PP_START_YEAR "1993" - &PP_END_YEAR "2019" @@ -47,7 +46,17 @@ fre_properties: - &GRID_SPEC "/archive/acr/mom6_input/nwa12/nwa12_grid_75z.tar" - + # Compile properties + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &F2003_FLAGS " -DINTERNAL_FILE_NML -g " + - &momIncludes "-Imom6/src/MOM6/pkg/CVMix-src/include" + - &sisincludes "-Imom6/src/MOM6/src/framework" + - &MOM6_GIT_FIX "main" + - &FMS_GIT_TAG "2024.01.02" + - &CPL_GIT_TAG "2024.01.02" + - &INTEL "intel-classic" + + shared: directories: &shared_directories history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] @@ -77,6 +86,21 @@ shared: compileYaml: &compile_yaml platformYaml: +################################################################################################# +# COMPILE # +################################################################################################# + +## The location for yamls +build: + platformYaml: platforms.yaml + compileYaml: compile.yaml +## User defined properties for building +#release: "2024.01" + +################################################################################################# +# POST-PROCESSING # +################################################################################################# + experiments: - name: "MOM6_SIS2_GENERIC_4P_compile_symm" pp: diff --git a/yamls/NWA12/compile.yaml b/yamls/NWA12/compile.yaml new file mode 100644 index 000000000..e204f37de --- /dev/null +++ b/yamls/NWA12/compile.yaml @@ -0,0 +1,65 @@ +compile: + experiment: "mom6_sis2_generic_4p_compile_symm_yaml" + container_addlibs: + baremetal_linkerflags: + src: + - component: FMS + paths: [FMS] + repo: "https://github.com/NOAA-GFDL/FMS.git" + branch: *FMS_GIT_TAG + cppdefs: " -DINTERNAL_FILE_NML -g -Duse_libMPI -Duse_netCDF -Duse_yaml -DMAXFIELDMETHODS_=600" #this uses deprecated IO + otherFlags: *FMSincludes + + - component: mom6 + requires: [FMS] + repo: "https://github.com/NOAA-GFDL/CEFI-regional-MOM6.git" + branch: *MOM6_GIT_FIX + paths: [mom6/src/MOM6/config_src/memory/dynamic_symmetric, + mom6/src/MOM6/config_src/drivers/FMS_cap, + mom6/src/MOM6/src/*/, + mom6/src/MOM6/src/*/*/, + mom6/src/MOM6/config_src/external/ODA_hooks, + mom6/src/MOM6/config_src/external/stochastic_physics, + mom6/src/MOM6/config_src/external/drifters, + mom6/src/MOM6/config_src/external/database_comms, + mom6/src/ocean_BGC/generic_tracers, + mom6/src/ocean_BGC/mocsy/src, + mom6/src/MOM6/pkg/GSW-Fortran/modules, + mom6/src/MOM6/pkg/GSW-Fortran/toolbox, + mom6/src/MOM6/config_src/infra/FMS2] + #makeOverrides: 'OPENMP=""' + otherFlags: !join [*FMSincludes," ", *momIncludes] + cppdefs: "-DINTERNAL_FILE_NML -g -DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DUSE_FMS2_IO + -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2 " + + - component: sis2 + requires: [FMS, mom6] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + paths: [mom6/src/SIS2/config_src/dynamic_symmetric, + mom6/src/SIS2/config_src/external/Icepack_interfaces, + mom6/src/SIS2/src, + mom6/src/icebergs/src, + mom6/src/ice_param] + otherFlags: !join [*FMSincludes," ", *momIncludes," ", *sisincludes] + #makeOverrides: 'OPENMP=""' + cppdefs: -DINTERNAL_FILE_NML -g -DUSE_FMS2_IO + + - component: land_null + requires: [FMS] + repo: "https://github.com/NOAA-GFDL/land_null.git" + paths: [mom6/src/land_null] + + - component: atmos_null + requires: [FMS] + repo: "https://github.com/NOAA-GFDL/atmos_null.git" + paths: [ mom6/src/atmos_null ] + otherFlags: !join [*FMSincludes," ", *momIncludes] + cppdefs: -DINTERNAL_FILE_NML -g + + - component: coupler + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + requires: [FMS, mom6, sis2, land_null, atmos_null] + paths: [mom6/src/coupler/shared, + mom6/src/coupler/full ] + otherFlags: !join [*FMSincludes," ", *momIncludes] + cppdefs: -DINTERNAL_FILE_NML -g -DUSE_FMS2_IO -D_USE_LEGACY_LAND_ -Duse_AM3_physics diff --git a/yamls/NWA12/platforms.yaml b/yamls/NWA12/platforms.yaml new file mode 100644 index 000000000..9b5cd2431 --- /dev/null +++ b/yamls/NWA12/platforms.yaml @@ -0,0 +1,49 @@ +platforms: + - name: ncrc5.intel22 + compiler: intel + modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] + modules: [ !join [*INTEL,"/2022.2.1"],"fre/bronx-22",cray-hdf5, cray-netcdf] + #fc: ftn + #cc: cc + mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc5/", *INTEL, ".mk" ] + modelRoot: /gpfs/f5/cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + + - name: ncrc5.intel23 + compiler: intel + modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] + modules: [ !join [*INTEL, "/2023.1.0"],"fre/bronx-22",cray-hdf5, cray-netcdf] + #fc: ftn + #cc: cc + mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc5/", *INTEL, ".mk"] + modelRoot: /gpfs/f5/cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + + - name: ncrc6.intel22 + compiler: intel + modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] + modules: [ !join [*INTEL,"/2022.2.1"],"fre/bronx-22",cray-hdf5, cray-netcdf] + #fc: ftn + #cc: cc + mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc6/", *INTEL, ".mk" ] + modelRoot: /gpfs/f6/ira-cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + + - name: ncrc6.intel23 + compiler: intel + modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] + modules: [ !join [*INTEL, "/2023.1.0"],"fre/bronx-22",cray-hdf5, cray-netcdf] + #fc: ftn + #cc: cc + mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc6/", *INTEL, ".mk"] + modelRoot: /gpfs/f6/ira-cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + + - name: hpcme.2023 + compiler: intel + RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] + modelRoot: /apps + #fc: mpiifort + #cc: mpiicc + container: true + containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" + containerBuild: "podman" + containerRun: "apptainer" + # NOTE: c5 and c6 make templates are currently the same. Defaulting to c6 for container runs + mkTemplate: "/apps/mkmf/templates/hpcme-intel23.mk" From 2f32bba83eb55f26f183126fb9c1b1e78bf68ab8 Mon Sep 17 00:00:00 2001 From: Utheri Wagura Date: Fri, 10 Jan 2025 10:25:35 -0500 Subject: [PATCH 2/4] Added yaml README and add comments to experiment yaml explaining how to run fre make --- yamls/NWA12/CEFI_NWA12_cobalt.yaml | 41 ++++++++--- yamls/NWA12/README.md | 112 +++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 yamls/NWA12/README.md diff --git a/yamls/NWA12/CEFI_NWA12_cobalt.yaml b/yamls/NWA12/CEFI_NWA12_cobalt.yaml index 80c5f2ceb..fa69019e7 100644 --- a/yamls/NWA12/CEFI_NWA12_cobalt.yaml +++ b/yamls/NWA12/CEFI_NWA12_cobalt.yaml @@ -1,14 +1,39 @@ -# POSTPROCESSING GUIDE +# USAGE +# +# The following yaml is designed to compile and postprocess the ocean_monthly component of the CEFI_NWA12_COBALT_V1 experiment +# Full Documentation is available here: https://noaa-gfdl.github.io/fre-cli/index.html +# +# 0.) Begin by setting up your environment by calling module load fre/2025.01 +# +# For modularity and compatibility with with cylc workflow engine, fre/2025.01 splits up the compile and post processcing process +# into several sub steps. # -# The following yaml is designed to postprocess the ocean_monthly component of the 2024_06 version of the CEFI_NWA12_COBALT_V1 experiment -# if postprocessing data from a different version of this experiment, be sure to change the FRE_STEM variable before postprocessing +# COMPILE GUIDE - BAREMETAL RUN +# +# 1.) Create script that will be used to checkout model components +# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod +# +# 2.) Create Makefile +# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod # -# Quick start instructions and source code can be found here: https://github.com/NOAA-GFDL/fre-cli/tree/main/fre/pp +# 3.) Create script that will compile the model, and OPTIONALLY run it with the --execute flag. You can manually run the +# compile script after this step as well +# fre make create-compile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute # -# 0.) Load module on GFDL Analysis: module load fre/canopy +# COMPILE GUIDE - CONTAINERS # -# For modularity and compatibility with with cylc workflow engine, fre/canopy splits up the post processcing process into several sub steps: +# 1.) Create checkout script with the "no parallel checkouts option" - this is required for container builds +# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --npc +# +# 2.) Create Makefile AND a Docker file. You can then build the container by manually running the createContainer.sh +# script, or by OPTIONALLY including the --execute file when creating the container +# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod +# fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute +# +# POSTPROCESSING GUIDE # +# 0.) Load module on GFDL Analysis: module load fre/2025.01 +# # 1.) Checkout the git repo containing postprocessing scripts and related files with the folowing command: # fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod # @@ -16,7 +41,7 @@ # fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y CEFI_NWA12_cobalt.yaml # fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y combined-CEFI_NWA12_COBALT_V1.yaml # -# 2a.) Fre/canopy does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator: +# 2a.) fre/2025.01 does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator: # mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/pp # # 2b.) (OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors @@ -36,7 +61,7 @@ fre_properties: - - &FRE_STEM "fre/cefi/NWA/2025_01" + - &FRE_STEM "fre/cefi/NWA/2024_11" - &PP_START_YEAR "1993" - &PP_END_YEAR "2019" diff --git a/yamls/NWA12/README.md b/yamls/NWA12/README.md new file mode 100644 index 000000000..a14ad2988 --- /dev/null +++ b/yamls/NWA12/README.md @@ -0,0 +1,112 @@ +# FRE yamls + +The latest version of the FMS Runtime Environment uses yamls instead of xmls to execute the FRE workflow. This folder contains experiment, compile, and platform yamls that can be used with the latest version of `fre` to compile and post process the `CEFI_NWA12_Cobalt_V1` experiment. The compiled experiment can then be run in conjuction with `fre/bronx-23` and the existing xmls in the `xml/NWA12` directory. + +The following sections assume that you are compiling and running experiments on Gaea C5, and then conducting postprocessing on GFDL's PPAN system. The `platforms.yaml` does support compilation on C6, but `bronx-23` does not support containerized runs on C6 - baremetal runs, however, are supported on C6. + +## Compile the Experiment. + +Begin by loading the appropriate modules on Gaea C5: +``` +module use -a /ncrc/home2/fms/local/modulefiles +module load fre/2025.01 +``` + +`fre/2025.01` supports both containerized compilation and bare-metal compilation, with very similar steps for both targets. + +### Baremetal Compilation + +Run the following command create the script that will be used to checkout the model components: +``` +fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod +``` + +Create the Makefile for the experiment: +``` +fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod +``` + +Create the script that actually compiles the experiment, and optionally run the script by passing in the --execute flag +``` +fre make create-compile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute +``` + +### Containerized Compilation + +Compiling the experiment into a container is a similar process. The only changes are 1.) passing in the --npc flag during the checkout step to prevent parallel checkouts, 2.) changing you platform to `hpcme.2023` and 3.) creating a Dockerfile and container creation script instead of a compilation script. Note that the Dockerfile will compile the model within the container as part of the build process. + +``` +fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --npc +fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod +fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute +``` + +## Running the Experiment + +`fre/2025.01` does not currently support running experiments with yamls, so in order to run the compiled experiment in fre, you will have to use the existing xmls along with `fre/bronx`. Make sure the `FRE_STEM` set in the xml matches the `FRE_STEM` set in the experiment yaml. + +### Baremetal Run + +Baremetal experiments can be run using the same run [instructions](https://github.com/NOAA-GFDL/CEFI-regional-MOM6/tree/main/xmls) available in the xmls directory. Be sure to load the modules described in that page before running `frerun`. + +### Containerized Run +Current xmls can be used for containerized runs, as well, so long as the following changes are made: + +1.) Run `module load fre/bronx-23` and change the `FRE_VERSION` property of the xml to `bronx-23`. +2.) Right below the `` tag, add a tag with a path to your `.sif` singularity image file: + ``` + + ``` + +With these changes, you should be able to run the experiment by calling `frerun` with an extra `--container` flag +``` +frerun -x CEFI_NWA12_cobalt.xml -p ncrc5.intel23 -t prod CEFI_NWA12_COBALT_V1 --container +``` + + +## Postprocessing the Experiment +Like `fre make`, postprocessing is split into several subcommands to improve modularity. On PPAN, load the appropriate module: +``` +module load fre/2025.01 +``` + +Checkout the git repo containing postprocessing scripts and related files with the folowing command: +``` +fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +``` + +Combine your main yaml and experiment yamls into a single yaml, then set up the cylc-src dir with the configure-yaml command: +``` +fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y CEFI_NWA12_cobalt.yaml +fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y combined-CEFI_NWA12_COBALT_V1.yaml +``` + +`fre/2025.01` does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator: +``` +mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/pp +``` + +(OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors +``` +tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel22__prod/history-manifest +``` + +Validate that all configuration files are correct +``` +fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +``` + +Create the cylc-run directory containing the final version of configuration files, scripts, and output directories +``` +fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +``` + +Run post processing +``` +fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +``` + +To monitor the status of each post processing step, run the following command: +``` +fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +``` From 8ade480f556626485e67cceeb1d08e5d7aaa8461 Mon Sep 17 00:00:00 2001 From: Utheri Wagura Date: Tue, 14 Jan 2025 16:04:47 -0500 Subject: [PATCH 3/4] Addressed comments in first review of PR #131, removed commented out code --- yamls/NWA12/CEFI_NWA12_cobalt.yaml | 37 +++++++++++++++--------------- yamls/NWA12/README.md | 35 +++++++++++++++++----------- yamls/NWA12/platforms.yaml | 32 ++++---------------------- 3 files changed, 44 insertions(+), 60 deletions(-) diff --git a/yamls/NWA12/CEFI_NWA12_cobalt.yaml b/yamls/NWA12/CEFI_NWA12_cobalt.yaml index fa69019e7..80f29c893 100644 --- a/yamls/NWA12/CEFI_NWA12_cobalt.yaml +++ b/yamls/NWA12/CEFI_NWA12_cobalt.yaml @@ -3,27 +3,28 @@ # The following yaml is designed to compile and postprocess the ocean_monthly component of the CEFI_NWA12_COBALT_V1 experiment # Full Documentation is available here: https://noaa-gfdl.github.io/fre-cli/index.html # -# 0.) Begin by setting up your environment by calling module load fre/2025.01 +# 0.) Set up your environment by running "module use -a /ncrc/home2/fms/local/modulefiles" followed by "module load fre/2025.01" # # For modularity and compatibility with with cylc workflow engine, fre/2025.01 splits up the compile and post processcing process # into several sub steps. # # COMPILE GUIDE - BAREMETAL RUN # -# 1.) Create script that will be used to checkout model components -# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod +# 1.) Create script that will be used to checkout model components and OPTIONALLY run it with the --execute flag. You can manually run the +# checkout script after this step as well +# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute # # 2.) Create Makefile -# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod +# fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod # # 3.) Create script that will compile the model, and OPTIONALLY run it with the --execute flag. You can manually run the # compile script after this step as well -# fre make create-compile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute +# fre make create-compile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute # # COMPILE GUIDE - CONTAINERS # # 1.) Create checkout script with the "no parallel checkouts option" - this is required for container builds -# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --npc +# fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod -npc # # 2.) Create Makefile AND a Docker file. You can then build the container by manually running the createContainer.sh # script, or by OPTIONALLY including the --execute file when creating the container @@ -35,33 +36,33 @@ # 0.) Load module on GFDL Analysis: module load fre/2025.01 # # 1.) Checkout the git repo containing postprocessing scripts and related files with the folowing command: -# fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +# fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod # # 2.) Combine your main yaml and experiment yamls into a single yaml, then set up the cylc-src dir with the configure-yaml command: -# fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y CEFI_NWA12_cobalt.yaml -# fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y combined-CEFI_NWA12_COBALT_V1.yaml +# fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml +# fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml --use pp # # 2a.) fre/2025.01 does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator: -# mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/pp +# mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/pp # # 2b.) (OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors -# tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel22__prod/history-manifest +# tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel23__prod/history-manifest # # 3.) Validate that all configuration files are correct -# fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +# fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod # # 4.) Create the cylc-run directory containing the final version of configuration files, scripts, and output directories -# fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +# fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod # # 5.) Run post processing -# fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +# fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod # # 6.) To monitor the status of each post processing step, run the following command: -# fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +# fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod fre_properties: - - &FRE_STEM "fre/cefi/NWA/2024_11" + - &FRE_STEM "fre/cefi/NWA/ShortCanopyTest" - &PP_START_YEAR "1993" - &PP_END_YEAR "2019" @@ -77,8 +78,8 @@ fre_properties: - &momIncludes "-Imom6/src/MOM6/pkg/CVMix-src/include" - &sisincludes "-Imom6/src/MOM6/src/framework" - &MOM6_GIT_FIX "main" - - &FMS_GIT_TAG "2024.01.02" - - &CPL_GIT_TAG "2024.01.02" + - &FMS_GIT_TAG "2024.03" + - &CPL_GIT_TAG "2024.03" - &INTEL "intel-classic" diff --git a/yamls/NWA12/README.md b/yamls/NWA12/README.md index a14ad2988..43f7dc527 100644 --- a/yamls/NWA12/README.md +++ b/yamls/NWA12/README.md @@ -4,6 +4,8 @@ The latest version of the FMS Runtime Environment uses yamls instead of xmls to The following sections assume that you are compiling and running experiments on Gaea C5, and then conducting postprocessing on GFDL's PPAN system. The `platforms.yaml` does support compilation on C6, but `bronx-23` does not support containerized runs on C6 - baremetal runs, however, are supported on C6. +Full documentation for `Fre/2025.01` is available [here](https://noaa-gfdl.github.io/fre-cli/usage.html) + ## Compile the Experiment. Begin by loading the appropriate modules on Gaea C5: @@ -16,9 +18,10 @@ module load fre/2025.01 ### Baremetal Compilation -Run the following command create the script that will be used to checkout the model components: +Run the following command to create the script that will be used to checkout the model components, and optionally run the script by passing in the --execute flag. +If you don't provide the --execute flag, be sure to run the script printed at the end of this command manually. ``` -fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod +fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --execute ``` Create the Makefile for the experiment: @@ -33,10 +36,10 @@ fre make create-compile -y CEFI_NWA12_cobalt.yaml -p ncrc5.intel23 -t prod --exe ### Containerized Compilation -Compiling the experiment into a container is a similar process. The only changes are 1.) passing in the --npc flag during the checkout step to prevent parallel checkouts, 2.) changing you platform to `hpcme.2023` and 3.) creating a Dockerfile and container creation script instead of a compilation script. Note that the Dockerfile will compile the model within the container as part of the build process. +Compiling the experiment into a container is a similar process. The only changes are 1.) passing in the -npc flag during the checkout step to prevent parallel checkouts, 2.) changing you platform to `hpcme.2023` and 3.) creating a Dockerfile and container creation script instead of a compilation script. Note that the Dockerfile will compile the model within the container as part of the build process. ``` -fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --npc +fre make create-checkout -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod -npc fre make create-makefile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --execute ``` @@ -45,6 +48,10 @@ fre make create-dockerfile -y CEFI_NWA12_cobalt.yaml -p hpcme.2023 -t prod --exe `fre/2025.01` does not currently support running experiments with yamls, so in order to run the compiled experiment in fre, you will have to use the existing xmls along with `fre/bronx`. Make sure the `FRE_STEM` set in the xml matches the `FRE_STEM` set in the experiment yaml. +**NOTE**: Since capitilized names are not allowed in Dockerfiles, the current compile yaml names the compilation experiment `mom6_sis2_generic_4p_compile_symm_yaml` instead of `MOM6_SIS2_GENERIC_4P_compile_symm_yaml` like the xml. As a result, `fre` will print a warning that it cannot find your executable when you run `frerun`, and will eventually fail if you submit the job. + +To get around this, change the path set in the `executable` variable within the runscript printed by `frerun` to point to the exectable located in `${FRE_STEM}/mom6_sis2_generic_4p_compile_symm_yaml/exec` instead. + ### Baremetal Run Baremetal experiments can be run using the same run [instructions](https://github.com/NOAA-GFDL/CEFI-regional-MOM6/tree/main/xmls) available in the xmls directory. Be sure to load the modules described in that page before running `frerun`. @@ -68,45 +75,45 @@ frerun -x CEFI_NWA12_cobalt.xml -p ncrc5.intel23 -t prod CEFI_NWA12_COBALT_V1 -- Like `fre make`, postprocessing is split into several subcommands to improve modularity. On PPAN, load the appropriate module: ``` module load fre/2025.01 -``` +`` Checkout the git repo containing postprocessing scripts and related files with the folowing command: ``` -fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +fre pp checkout -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod ``` Combine your main yaml and experiment yamls into a single yaml, then set up the cylc-src dir with the configure-yaml command: ``` -fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y CEFI_NWA12_cobalt.yaml -fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod -y combined-CEFI_NWA12_COBALT_V1.yaml +fre yamltools combine-yamls -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml --use pp +fre pp configure-yaml -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod -y CEFI_NWA12_cobalt.yaml ``` `fre/2025.01` does not automatically create the pp dir for you, so you will have to mkdir this first to pass the validator: ``` -mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/pp +mkdir /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/pp ``` (OPTIONAL, BUT RECOMMENDED): Create a list of available tar files within your history tar archives to allow fre to catch a wider variety of errors ``` -tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel22-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel22__prod/history-manifest +tar -tf /archive/$USER/fre/cefi/NWA/2024_06/CEFI_NWA12_COBALT_V1/gfdl.ncrc5-intel23-prod/history/19930101.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/CEFI_NWA12_COBALT_V1__gfdl.ncrc5-intel23__prod/history-manifest ``` Validate that all configuration files are correct ``` -fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +fre pp validate -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod ``` Create the cylc-run directory containing the final version of configuration files, scripts, and output directories ``` -fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +fre pp install -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod ``` Run post processing ``` -fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +fre pp run -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod ``` To monitor the status of each post processing step, run the following command: ``` -fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel22 -t prod +fre pp status -e CEFI_NWA12_COBALT_V1 -p gfdl.ncrc5-intel23 -t prod ``` diff --git a/yamls/NWA12/platforms.yaml b/yamls/NWA12/platforms.yaml index 9b5cd2431..a9bf02d02 100644 --- a/yamls/NWA12/platforms.yaml +++ b/yamls/NWA12/platforms.yaml @@ -1,46 +1,22 @@ platforms: - - name: ncrc5.intel22 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL,"/2022.2.1"],"fre/bronx-22",cray-hdf5, cray-netcdf] - #fc: ftn - #cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc5/", *INTEL, ".mk" ] - modelRoot: /gpfs/f5/cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ - - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2023.1.0"],"fre/bronx-22",cray-hdf5, cray-netcdf] - #fc: ftn - #cc: cc + modules: [!join [*INTEL, /2023.2.0], fre/bronx-22, cray-hdf5, cray-netcdf] mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc5/", *INTEL, ".mk"] - modelRoot: /gpfs/f5/cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ - - - name: ncrc6.intel22 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL,"/2022.2.1"],"fre/bronx-22",cray-hdf5, cray-netcdf] - #fc: ftn - #cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc6/", *INTEL, ".mk" ] - modelRoot: /gpfs/f6/ira-cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + modelRoot: !join ["/gpfs/f5/cefi/scratch/${USER}", "/", *FRE_STEM ] - name: ncrc6.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2023.1.0"],"fre/bronx-22",cray-hdf5, cray-netcdf] - #fc: ftn - #cc: cc + modules: [!join [*INTEL, /2023.2.0], fre/bronx-22, cray-hdf5, cray-netcdf] mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-22/site/ncrc6/", *INTEL, ".mk"] - modelRoot: /gpfs/f6/ira-cefi/scratch/Utheri.Wagura/fre/cefi/NWA/Canopy_202412/ + modelRoot: !join ["/gpfs/f6/ira-cefi/scratch/${USER}", "/", *FRE_STEM ] - name: hpcme.2023 compiler: intel RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] modelRoot: /apps - #fc: mpiifort - #cc: mpiicc container: true containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" containerBuild: "podman" From 1bd018165b42780bf41e9179a74d320eef55734e Mon Sep 17 00:00:00 2001 From: Yi-Cheng Teng - NOAA GFDL <143743249+yichengt900@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:50:23 -0500 Subject: [PATCH 4/4] fixed format --- yamls/NWA12/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamls/NWA12/README.md b/yamls/NWA12/README.md index 43f7dc527..e16000765 100644 --- a/yamls/NWA12/README.md +++ b/yamls/NWA12/README.md @@ -75,7 +75,7 @@ frerun -x CEFI_NWA12_cobalt.xml -p ncrc5.intel23 -t prod CEFI_NWA12_COBALT_V1 -- Like `fre make`, postprocessing is split into several subcommands to improve modularity. On PPAN, load the appropriate module: ``` module load fre/2025.01 -`` +``` Checkout the git repo containing postprocessing scripts and related files with the folowing command: ```