diff --git a/.circleci/config.yml b/.circleci/config.yml index a75df8f..a007a8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,28 +1,63 @@ -version: 2 -jobs: - build: +version: 2.1 + +executors: + gcc-build-env: docker: - - image: gmao/geos-build-env-gcc-source:6.0.4 + - image: gmao/ubuntu20-geos-env-mkl:v6.1.0-openmpi_4.0.5-gcc_10.2.0 + auth: + username: $DOCKERHUB_USER + password: $DOCKERHUB_AUTH_TOKEN + environment: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_btl_vader_single_copy_mechanism: none + resource_class: large + #MEDIUM# resource_class: medium + +workflows: + version: 2.1 + build-test: + jobs: + - build-GEOSctm: + context: + - docker-hub-creds + +jobs: + build-GEOSctm: + executor: gcc-build-env working_directory: /root/project steps: - - checkout + - checkout: + path: GEOSctm - run: name: "Versions etc" command: mpirun --version && gfortran --version && echo $BASEDIR && pwd && ls - run: name: "Mepo clone external repos" command: | - mepo init + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm mepo clone mepo status + - run: + name: "Mepo checkout-if-exists" + command: | + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm + echo "${CIRCLE_BRANCH}" + if [ "${CIRCLE_BRANCH}" != "develop" ] && [ "${CIRCLE_BRANCH}" != "master" ] && [ "${CIRCLE_BRANCH}" != "main" ] + then + mepo checkout-if-exists ${CIRCLE_BRANCH} + fi + mepo status - run: name: "CMake" command: | + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm mkdir build cd build - cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DUSE_F2PY=OFF - run: name: "Build" command: | - cd build - make -j2 install + cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm/build + make -j"$(nproc)" install + #MEDIUM# make -j4 install diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e0d712..60263a3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,16 @@ # Order is important; the last matching pattern takes the most # precedence. +# Everything in GEOSgcm should be owned by the GCM Gatekeepers +* @GEOS-ESM/ctm-gatekeepers + +# The GCM gatekeepers and CMake should know/approve these +/.github/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.circleci/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.codebuild/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers + # The GEOS CMake Team is the CODEOWNER for the CMakeLists.txt files in this repository CMakeLists.txt @GEOS-ESM/cmake-team +# The GEOS CMake Team should be notified about and approve config changes +/config/ @GEOS-ESM/cmake-team diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 0000000..5a7d9b5 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,61 @@ +name: Build Tests + +on: + pull_request: + types: [opened, synchronize, reopened] + # Do not run if the only files changed cannot affect the build + paths-ignore: + - "**.md" + - "parallel_build.csh" + - ".github/CODEOWNERS" + - ".codebuild/**" + - ".circleci/**" + +jobs: + build_ctm: + name: Build GEOSctm + if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')" + runs-on: ubuntu-latest + container: + image: gmao/ubuntu20-geos-env-mkl:v6.1.0-openmpi_4.0.5-gcc_10.2.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + env: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_btl_vader_single_copy_mechanism: none + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.5.0 + with: + access_token: ${{ github.token }} + - name: Checkout CTM + uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Versions etc. + run: | + gfortran --version + mpirun --version + echo $BASEDIR + - name: Mepo clone external repos + run: | + mepo clone + mepo status + mepo status + - name: Update other branches + if: + "!contains('refs/heads/main,refs/heads/develop', github.ref)" + run: | + mepo checkout-if-exists ${GITHUB_HEAD_REF} + mepo status + - name: CMake + run: | + mkdir build + cd build + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF + - name: Build + run: | + cd build + make -j4 install diff --git a/.gitignore b/.gitignore index c7a54ff..78f284c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ *~ /@cmake/ -/@modules/ +/cmake/ +/cmake@/ /@env/ -/BUILD/ -/build*/ -/install*/ -parallel_build.o* +/env/ +/env@/ /.mepo/ +parallel_build.o* +log.* diff --git a/CMakeLists.txt b/CMakeLists.txt index 2706b68..9e68486 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW) project ( GEOSctm - VERSION 1.0 + VERSION 2.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}") @@ -25,9 +25,7 @@ set (DOING_GEOS5 YES) # Should find a better place for this - used in Chem component set (ACG_FLAGS -v) -set (F2PYEXT .so) -set (F2PY_SUFFIX .so) -set (FV_PRECISION R8) +set (FV_PRECISION "R8" CACHE STRING "Precision of FV3 core (R4, R4R8, R8)") list (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/@cmake") include (esma) @@ -44,3 +42,16 @@ include_directories(${MPI_Fortran_INCLUDE_PATH}) add_subdirectory (@env) add_subdirectory (src) +# https://www.scivision.dev/cmake-auto-gitignore-build-dir/ +# --- auto-ignore build directory +if(NOT EXISTS ${PROJECT_BINARY_DIR}/.gitignore) + file(WRITE ${PROJECT_BINARY_DIR}/.gitignore "*") +endif() + +# Piggyback that file into install +install( + FILES ${PROJECT_BINARY_DIR}/.gitignore + DESTINATION ${CMAKE_INSTALL_PREFIX} + ) + + diff --git a/Externals.cfg b/Externals.cfg deleted file mode 100644 index dfa8bc0..0000000 --- a/Externals.cfg +++ /dev/null @@ -1,54 +0,0 @@ -[ESMA_env] -required = True -repo_url = git@github.com:GEOS-ESM/ESMA_env.git -local_path = ./@env -tag = v2.0.2 -protocol = git - -[ESMA_cmake] -required = True -repo_url = git@github.com:GEOS-ESM/ESMA_cmake.git -local_path = ./@cmake -tag = v2.1.2 -externals = Externals.cfg -protocol = git - -[GMAO_Shared] -required = True -repo_url = git@github.com:GEOS-ESM/GMAO_Shared.git -local_path = ./src/Shared/@GMAO_Shared -tag = v1.1.0 -protocol = git -sparse = ../../../config/GMAO_Shared.sparse - -[MAPL] -required = True -repo_url = git@github.com:GEOS-ESM/MAPL.git -local_path = ./src/Shared/@MAPL -tag = v2.0.0 -protocol = git - -[FVdycoreCubed_GridComp] -required = True -repo_url = git@github.com:GEOS-ESM/FVdycoreCubed_GridComp.git -local_path = ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp -tag = v1.1.0 -externals = Externals.cfg -protocol = git - -[GEOSchem_GridComp] -required = True -repo_url = git@github.com:GEOS-ESM/GEOSchem_GridComp.git -local_path = ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp -tag = v1.2.0 -protocol = git - -[FMS] -required = True -repo_url = git@github.com:GEOS-ESM/FMS.git -local_path = ./src/Shared/@FMS -tag = geos/2019.01.01 -protocol = git - -[externals_description] -schema_version = 1.0.0 diff --git a/README.md b/README.md index d77eb7a..7863a7e 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,9 @@ In your `.bashrc` or `.tcshrc` or other rc file add a line: -##### NCCS (SLES11) - +##### NCCS ``` -module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES11 +module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES12 ``` ##### NAS @@ -33,19 +32,74 @@ Now load the `GEOSenv` module: ``` module load GEOSenv ``` -which obtains the latest `git`, `CMake`, and `manage_externals` modules. +which obtains the latest `git`, `CMake`, and `mepo` modules. #### Obtain the Model +On GitHub, there are three ways to clone the model: SSH, HTTPS, or GitHub CLI. +The first two are "git protocols" which determine how `git` communicates with +GitHub: either through https or ssh. (The latter is a CLI that uses either ssh or +https protocol underneath.) + +For developers of GEOS, the SSH git protocol is recommended as it can avoid some issues if +[two-factor authentication +(2FA)](https://docs.github.com/en/github/authenticating-to-github/securing-your-account-with-two-factor-authentication-2fa) +is enabled on GitHub. + +##### SSH + +To clone the GEOSctm using the SSH url (starts with `git@github.com`), you run: ``` git clone git@github.com:GEOS-ESM/GEOSctm.git ``` +###### Permission denied (publickey) + +If this is your first time using GitHub with any SSH URL, you might get this +error: +``` +Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +``` + +If you do see this, you need to [upload an ssh +key](https://docs.github.com/en/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account) +to your GitHub account. This needs to be done on any machine that you want to +use the SSH URL through. + + +##### HTTPS + +To clone the model through HTTPS you run: + +``` +git clone https://github.com/GEOS-ESM/GEOSctm.git +``` + +Note that if you use the HTTPS URL and have 2FA set up on GitHub, you will need +to use [personal access +tokens](https://docs.github.com/en/github/authenticating-to-github/accessing-github-using-two-factor-authentication#authenticating-on-the-command-line-using-https) +as a password. + +##### GitHub CLI + +You can also use the [GitHub CLI](https://cli.github.com/) with: +``` +gh repo clone GEOS-ESM/GEOSctm +``` + +Note that when you first use `gh`, it will ask what your preferred git protocol +is (https or ssh) to use "underneath". The caveats above will apply to whichever +you choose. + --- ### Single Step Building of the Model -If all you wish is to build the model, you can run `parallel_build.csh` from a head node. Doing so will checkout all the external repositories of the model and build it. When done, the resulting model build will be found in `build/` and the installation will be found in `install/` with setup scripts like `gcm_setup` and `fvsetup` in `install/bin`. +If all you wish is to build the model, you can run `parallel_build.csh` from a head node. Doing so will checkout all the external repositories of the model and build it. When done, the resulting model build will be found in `build/` and the installation will be found in `install/` with setup scripts like `ctm_setup` in `install/bin`. #### Debug Version of GEOS @@ -57,12 +111,21 @@ To obtain a debug version, you can run `parallel_build.csh -debug` which will bu The steps detailed below are essentially those that `parallel_build.csh` performs for you. Either method should yield identical builds. -##### Checkout externals +#### Mepo + +The GEOS CTM is comprised of a set of sub-repositories. These are +managed by a tool called [mepo](https://github.com/GEOS-ESM/mepo). To +clone all the sub-repos, you can run `mepo clone` inside the fixture: + ``` cd GEOSctm -checkout_externals +mepo clone ``` +The first command initializes the multi-repository and the second one +clones and assembles all the sub-repositories according to +`components.yaml` + #### Build the Model ##### Load Compiler, MPI Stack, and Baselibs @@ -96,6 +159,14 @@ and CMake will install there. ##### Build and Install with Make ``` -make -j6 install +make -jN install ``` +where `N` is the number of parallel processes. On discover head nodes, this should only be as high as 2 due to limits on the head nodes. On a compute node, you can set `N` has high as you like, though 8-12 is about the limit of parallelism in our model's make system. +### Run the CTM + +Once the model has built successfully, you will have an `install/` directory in your checkout. To run `ctm_setup` go to the `install/bin/` directory and run it there: +``` +cd install/bin +./ctm_setup +``` diff --git a/components.yaml b/components.yaml index 6c27da9..dfaf1af 100644 --- a/components.yaml +++ b/components.yaml @@ -1,53 +1,76 @@ +GEOSctm: + fixture: true + develop: main + env: - local: ./@env - remote: ../ESMA_env.git - tag: v2.0.2 - develop: master + local: ./@env + remote: ../ESMA_env.git + tag: v3.2.1 + develop: main cmake: - local: ./@cmake - remote: ../ESMA_cmake.git - tag: v2.1.2 - develop: develop + local: ./@cmake + remote: ../ESMA_cmake.git + tag: v3.4.2 + develop: develop ecbuild: - local: ./@cmake/@ecbuild - remote: ../ecbuild.git - branch: feature/FindMKL-portability-improvement + local: ./@cmake/@ecbuild + remote: ../ecbuild.git + tag: geos/v1.0.6 GMAO_Shared: - local: ./src/Shared/@GMAO_Shared - remote: ../GMAO_Shared.git - tag: v1.1.0 - sparse: ./config/GMAO_Shared.sparse - develop: master + local: ./src/Shared/@GMAO_Shared + remote: ../GMAO_Shared.git + tag: v1.4.1 + sparse: ./config/GMAO_Shared.sparse + develop: main MAPL: - local: ./src/Shared/@MAPL - remote: ../MAPL.git - tag: v2.0.0 - develop: develop + local: ./src/Shared/@MAPL + remote: ../MAPL.git + tag: v2.6.7 + develop: develop FMS: - local: ./src/Shared/@FMS - remote: ../FMS.git - tag: geos/2019.01.01 - develop: geos/master + local: ./src/Shared/@FMS + remote: ../FMS.git + tag: geos/2019.01.02+noaff.7 + develop: geos/release/2019.01 FVdycoreCubed_GridComp: - local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp - remote: ../FVdycoreCubed_GridComp.git - tag: v1.1.0 - develop: develop + local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp + remote: ../FVdycoreCubed_GridComp.git + tag: v1.2.13 + develop: develop fvdycore: - local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore - remote: ../GFDL_atmos_cubed_sphere.git - tag: geos/v1.1.0 - develop: geos/develop + local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore + remote: ../GFDL_atmos_cubed_sphere.git + tag: geos/v1.1.6 + develop: geos/develop GEOSchem_GridComp: - local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp - remote: ../GEOSchem_GridComp.git - tag: v1.2.0 - develop: develop + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp + remote: ../GEOSchem_GridComp.git + tag: v1.6.0 + develop: develop + +HEMCO: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/Shared/HEMCO/@HEMCO + remote: ../HEMCO.git + tag: geos/v2.2.1 + develop: geos/develop + +geos-chem: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/GEOSCHEMchem_GridComp/@geos-chem + remote: ../geos-chem.git + tag: geos/v13.0.0-rc1 + develop: geos/develop + +GOCART: + local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/@GOCART + remote: ../GOCART.git + sparse: ./config/GOCART.sparse + tag: v1.0.1 + develop: develop diff --git a/config/GEOSgcm_GridComp.sparse b/config/GEOSgcm_GridComp.sparse deleted file mode 100644 index 5c7cc60..0000000 --- a/config/GEOSgcm_GridComp.sparse +++ /dev/null @@ -1,6 +0,0 @@ -/GEOSagcm_GridComp/GEOSsuperdyn_GridComp/FVdycoreCubed_GridComp -/GEOSagcm_GridComp/GEOSsuperdyn_GridComp/CMakeLists.txt -/GEOSagcm_GridComp/CMakeLists.txt -/CMakeLists.txt - - diff --git a/config/GOCART.sparse b/config/GOCART.sparse new file mode 100644 index 0000000..c7fc83d --- /dev/null +++ b/config/GOCART.sparse @@ -0,0 +1,10 @@ + +/* +!/CCPP +!/config +!/ESMF/Aerosol_GridComp +!/ESMF/Apps +!/ESMF/GOCART2G_GridComp +!/ESMF/NUOPC +!/ESMF/Shared/Chem_Shared +!/ESMF/Shared/Chem_Base diff --git a/parallel_build.csh b/parallel_build.csh index 7511179..b1aa7d3 100755 --- a/parallel_build.csh +++ b/parallel_build.csh @@ -20,12 +20,11 @@ setenv ESMADIR $srcdir # tcsh script maker set origargv = "$argv" -setenv external "" -setenv USEMEPO FALSE -while ($#argv) +setenv DEVELOP FALSE - if ("$1" == "-mepo") then - setenv USEMEPO TRUE +while ($#argv) + if ("$1" == "-develop") then + setenv DEVELOP TRUE endif shift @@ -33,19 +32,23 @@ end if (! -d ${ESMADIR}/@env) then if ($?PBS_JOBID || $?SLURM_JOBID) then - echo " checkout_externals must be run!" + echo " mepo clone must be run!" echo " This requires internet access but you are on a compute node" echo " Please run from a head node" exit 1 else - if ( "$USEMEPO" == "TRUE") then echo "Running mepo initialization" mepo init mepo clone - else - echo " Running checkout_externals" - checkout_externals $external + if ( "$DEVELOP" == "TRUE" ) then + echo "Checking out development branches of GEOSgcm_GridComp and GEOSgcm_App" + mepo develop GEOSgcm_GridComp GEOSgcm_App + endif endif +else + if ( "$DEVELOP" == "TRUE" ) then + echo "Checking out development branches of GEOSgcm_GridComp and GEOSgcm_App" + mepo develop GEOSgcm_GridComp GEOSgcm_App endif endif diff --git a/src/Applications/GEOSctm_App/.GEOSCTM_VERSION b/src/Applications/GEOSctm_App/.GEOSCTM_VERSION index f1d9095..e403afd 100644 --- a/src/Applications/GEOSctm_App/.GEOSCTM_VERSION +++ b/src/Applications/GEOSctm_App/.GEOSCTM_VERSION @@ -1 +1 @@ -Icarus-3_2_CTM-r1 +GEOSctm-v@CMAKE_PROJECT_VERSION@ diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index ca481c6..da72f8b 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -11,15 +11,9 @@ target_link_libraries (GEOSctm.x ${OpenMP_Fortran_LIBRARIES}) set_target_properties(GEOSctm.x PROPERTIES LINK_FLAGS "${OpenMP_Fortran_FLAGS}") target_include_directories (GEOSctm.x PUBLIC ${INC_ESMF}) -file (GLOB templates *.tmpl) - -set(otheretc - fvcore_layout.rc - .GEOSCTM_VERSION - ) +file (GLOB templates CONFIGURE_DEPENDS *.tmpl) set (programs - ctm_setup ctm_run.j ctm_regress.j ctm_post.j @@ -29,9 +23,8 @@ set (programs ctm_moveplot.j ) - install ( - FILES ${templates} ${otheretc} + FILES ${templates} DESTINATION etc ) @@ -41,7 +34,25 @@ install ( ) install ( - FILES fvcore_layout.rc + FILES fvcore_layout.rc logging.yaml DESTINATION etc ) + +if(HYDROSTATIC) + set(CFG_HYDROSTATIC TRUE) +else() + set(CFG_HYDROSTATIC FALSE) +endif() + +set (setup_scripts + ctm_setup + ) + +foreach (file ${setup_scripts}) + configure_file(${file} ${file} @ONLY) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file} DESTINATION bin) +endforeach () + +configure_file(.GEOSCTM_VERSION .GEOSCTM_VERSION @ONLY) +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.GEOSCTM_VERSION DESTINATION etc) diff --git a/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl b/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl index 3db95c7..e38770a 100644 --- a/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl +++ b/src/Applications/GEOSctm_App/GEOSCTM.rc.tmpl @@ -3,6 +3,8 @@ # ---------------------------------------- NX: @NX NY: @NY + IOSERVER_NODES: @IOS_NDS + IM: @GEOSCTM_IM JM: @GEOSCTM_JM LM: @GEOSCTM_LM @@ -43,8 +45,8 @@ PRINTRC: 1 # Set the number of parallel I/O processes to use when # RESTART_TYPE and or CHECKPOINT_TYPE are set to pbinary or pnc4 #--------------------------------------------------------------- -PARALLEL_READFORCING: 1 -NUM_READERS: 1 +PARALLEL_READFORCING: 0 +NUM_READERS: @NUM_READERS NUM_WRITERS: 1 MAPL_ENABLE_BOOTSTRAP: @BOOT diff --git a/src/Applications/GEOSctm_App/GEOSctm.F90 b/src/Applications/GEOSctm_App/GEOSctm.F90 index f09c204..da75b3f 100644 --- a/src/Applications/GEOSctm_App/GEOSctm.F90 +++ b/src/Applications/GEOSctm_App/GEOSctm.F90 @@ -14,7 +14,7 @@ Program GEOSctm ! !USES: - use MAPL_Mod + use MAPL use GEOS_ctmGridCompMod, only: ROOT_SetServices => SetServices implicit NONE diff --git a/src/Applications/GEOSctm_App/ctm_archive.j b/src/Applications/GEOSctm_App/ctm_archive.j index b339af5..fd9975e 100644 --- a/src/Applications/GEOSctm_App/ctm_archive.j +++ b/src/Applications/GEOSctm_App/ctm_archive.j @@ -4,12 +4,12 @@ # Batch Parameters for Archive Job ####################################################################### -#PBS -l walltime=@ARCHIVE_T +#@BATCH_TIME@ARCHIVE_T #@ARCHIVE_P -#PBS -N @ARCHIVE_N +#@BATCH_JOBNAME@ARCHIVE_N #@ARCHIVE_Q #@BATCH_GROUP -#PBS -o OUTPUT +#@BATCH_OUTPUTNAMEOUTPUT ####################################################################### # System Environment Variables diff --git a/src/Applications/GEOSctm_App/ctm_convert.j b/src/Applications/GEOSctm_App/ctm_convert.j index bf43c2c..3342314 100755 --- a/src/Applications/GEOSctm_App/ctm_convert.j +++ b/src/Applications/GEOSctm_App/ctm_convert.j @@ -9,9 +9,9 @@ # Batch Parameters for Run Job ####################################################################### -#PBS -l walltime=@CONVERT_T +#@BATCH_TIME@CONVERT_T #@CONVERT_P -#PBS -N @CONVERT_N +#@BATCH_JOBNAME@CONVERT_N #@RUN_Q #@BATCH_GROUP @@ -25,6 +25,10 @@ limit stacksize unlimited @SETENVS +# Set OMP_NUM_THREADS +# ------------------- +setenv OMP_NUM_THREADS 1 + ####################################################################### # Architecture Specific Environment Variables ####################################################################### @@ -32,13 +36,15 @@ limit stacksize unlimited setenv ARCH `uname` setenv SITE @SITE +setenv GEOSDIR @GEOSDIR setenv GEOSBIN @GEOSBIN -setenv RUN_CMD "@RUN_CMD" setenv GCMVER @GCMVER echo "Sourcing g5_modules in $GEOSBIN" source $GEOSBIN/g5_modules >& /dev/null -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib + +setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " ####################################################################### # Experiment Specific Environment Variables @@ -71,25 +77,26 @@ if (! -e $ORGDIR) mkdir -p $ORGDIR # Set Experiment Run Parameters ####################################################################### -set NX = `grep "^ *NX:" $HOMDIR/AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY:" $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set AGCM_LM = `grep AGCM_LM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_IM = `grep OGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set OGCM_JM = `grep OGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>FVLATLON<<>>FVCUBED<<>>FVCUBED<<>>DATAOCEAN<<>>COUPLED<<& /dev/null - /bin/ln -sf $EXPDIR/RC/* . - /bin/cp -f $HOMDIR/*.rc . + +/bin/ln -sf $EXPDIR/RC/* . +@CPEXEC -f $HOMDIR/*.rc . +@CPEXEC -f $HOMDIR/*.nml . +@CPEXEC -f $HOMDIR/*.yaml . + +cat fvcore_layout.rc >> input.nml + if(-e ExtData.rc ) /bin/rm -f ExtData.rc set extdata_files = `/bin/ls -1 *_ExtData.rc` @@ -112,7 +125,7 @@ cat $extdata_files > ExtData.rc unsetenv FNDDIR -set fv_rst = `cat AGCM.rc | grep "^DYN_INTERNAL_RESTART_FILE" | cut -d ":" -f2` +set fv_rst = `cat GEOSCTM.rc | grep "^DYN_INTERNAL_RESTART_FILE" | cut -d ":" -f2` if (-e $CNVDIR/$fv_rst ) then setenv FNDDIR $CNVDIR @@ -164,7 +177,7 @@ echo "Converting from $fromformat restarts to $toformat restarts" echo "Copying cap_restart into $SCRDIR for BCs" if (-e $FNDDIR/cap_restart) then - /bin/cp $FNDDIR/cap_restart . + @CPEXEC $FNDDIR/cap_restart . else echo "Couldn't find cap_restart in $FNDDIR. Please place" echo "cap_restart in the same directory as the restarts." @@ -178,10 +191,11 @@ endif setenv BCSDIR @BCSDIR setenv SSTDIR @SSTDIR setenv CHMDIR @CHMDIR -setenv BCRSLV ${ATMOStag}_${OCEANtag} +>>>DATAOCEAN<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<< $FILE /bin/mkdir -p ExtData /bin/ln -sf $CHMDIR/* ExtData ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< $FILE >>>FVCUBED<<>>FVCUBED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<<@CPEXEC $HOMDIR/*_table . +>>>COUPLED<<<@CPEXEC $GRIDDIR/INPUT/* INPUT >>>COUPLED<<>>COUPLED<<>>FVCUBED<<>>FVCUBED<<>>FVCUBED<<>>COUPLED<<>>COUPLED<<<@CPEXEC $CNVDIR/RESTART/* INPUT ####################################################################### # Set Experiment Run Parameters that were altered ####################################################################### -set NX = `grep "^ *NX:" AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY:" AGCM.rc | cut -d':' -f2` +set NX = `grep "^ *NX:" GEOSCTM.rc | cut -d':' -f2` +set NY = `grep "^ *NY:" GEOSCTM.rc | cut -d':' -f2` # Check for Over-Specification of CPU Resources # --------------------------------------------- - if ($?PBS_NODEFILE) then + if ($?SLURM_NTASKS) then + set NCPUS = $SLURM_NTASKS + else if ($?PBS_NODEFILE) then set NCPUS = `cat $PBS_NODEFILE | wc -l` + else + set NCPUS = NULL + endif + + if ( $NCPUS != NULL ) then @ NPES = $NX * $NY if( $NPES > $NCPUS ) then echo "CPU Resources are Over-Specified" @@ -409,11 +479,11 @@ set NY = `grep "^ *NY:" AGCM.rc | cut -d':' -f2` if ( -x $GEOSBIN/rs_numtiles.x ) then - set N_SALT_TILES_EXPECTED = `/bin/grep '^\s*0' tile.data | wc -l` - set N_SALT_TILES_FOUND = `$GEOSBIN/rs_numtiles.x saltwater_internal_rst | awk '{print $3}'` + set N_SALT_TILES_EXPECTED = `grep '^ *0' tile.data | wc -l` + set N_SALT_TILES_FOUND = `$RUN_CMD 1 $GEOSBIN/rs_numtiles.x openwater_internal_rst | grep Total | awk '{print $NF}'` if ( $N_SALT_TILES_EXPECTED != $N_SALT_TILES_FOUND ) then - echo "Error! Found $N_SALT_TILES_FOUND tiles in saltwater. Expect to find $N_SALT_TILES_EXPECTED tiles." + echo "Error! Found $N_SALT_TILES_FOUND tiles in openwater. Expect to find $N_SALT_TILES_EXPECTED tiles." echo "Your restarts are probably for a different ocean." exit 7 endif @@ -455,7 +525,7 @@ endif ####################################################################### set numrst = `echo $found_rst_types | wc -w` -set numchk = `echo $chk_types | wc -w` +set numchk = `echo $chk_files | wc -w` @ n = 1 @ z = $numchk + 1 @@ -463,11 +533,8 @@ while ( $n <= $numrst ) if ( -e $found_rst_files[$n] ) then @ m = 1 while ( $m <= $numchk ) - if( $chk_types[$m] == $found_rst_types[$n] || ) then - set chk_ext = `cat AGCM.rc | grep ${chk_types[$m]}_CHECKPOINT_TYPE | cut -d: -f2` - if( $chk_ext =~ *nc4 ) set ext = nc4 - if( $chk_ext =~ *binary ) set ext = bin - /bin/mv $chk_files[$m] $RSTDIR/$found_rst_files[$n].$ext + if( $chk_files[$m] == $found_rst_types[$n] || ) then + /bin/mv $chk_file_names[$m] $RSTDIR/$found_rst_files[$n]$toext @ m = $numchk + 999 else @ m = $m + 1 @@ -475,7 +542,7 @@ while ( $n <= $numrst ) end wait if( $m == $z ) then - echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_types[$n] + echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_files[$n] exit endif endif diff --git a/src/Applications/GEOSctm_App/ctm_moveplot.j b/src/Applications/GEOSctm_App/ctm_moveplot.j index 5409e05..7efdfb1 100755 --- a/src/Applications/GEOSctm_App/ctm_moveplot.j +++ b/src/Applications/GEOSctm_App/ctm_moveplot.j @@ -4,12 +4,12 @@ # Batch Parameters for Plot Job ####################################################################### -#PBS -l walltime=1:00:00 -#PBS -l select=1:ncpus=1 -#PBS -N @MOVE_N +#@BATCH_TIME1:00:00 +#@MOVE_P +#@BATCH_JOBNAME@MOVE_N #@MOVE_Q #@BATCH_GROUP -#PBS -o ctm_moveplot.o +#@BATCH_OUTPUTNAMEctm_moveplot.o ####################################################################### # System Environment Variables @@ -29,7 +29,7 @@ setenv ARCH `uname` setenv SITE @SITE setenv GEOSBIN @GEOSBIN -setenv GEOSUTIL @GEOSSRC/GMAO_Shared/GEOS_Util +setenv GEOSUTIL @GEOSSRC source $GEOSBIN/g5_modules setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib diff --git a/src/Applications/GEOSctm_App/ctm_plot.tmpl b/src/Applications/GEOSctm_App/ctm_plot.tmpl index 6496cf5..718b4b7 100755 --- a/src/Applications/GEOSctm_App/ctm_plot.tmpl +++ b/src/Applications/GEOSctm_App/ctm_plot.tmpl @@ -4,12 +4,12 @@ # Batch Parameters for Plot Job ####################################################################### -#PBS -l walltime=@PLOT_T +#@BATCH_TIME@PLOT_T #@PLOT_P -#PBS -N @PLOT_N +#@BATCH_JOBNAME@NAME #@PLOT_Q #@BATCH_GROUP -#PBS -o OUTPUT +#@BATCH_OUTPUTNAMEOUTPUT ####################################################################### # System Environment Variables @@ -29,7 +29,7 @@ setenv ARCH `uname` setenv SITE @SITE setenv GEOSBIN @GEOSBIN -setenv GEOSUTIL @GEOSSRC/GMAO_Shared/GEOS_Util +setenv GEOSUTIL @GEOSSRC source $GEOSBIN/g5_modules setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib @@ -59,5 +59,5 @@ endif ####################################################################### set MOVE = `grep PLOT_MOVE: $EXPDIR/plot/plot.rc | cut -d'#' -f1 | cut -d':' -f2` -if( $MOVE[1] == 'ON' ) qsub $EXPDIR/plot/ctm_moveplot.j +if( $MOVE[1] == 'ON' ) @BATCH_CMD $EXPDIR/plot/ctm_moveplot.j diff --git a/src/Applications/GEOSctm_App/ctm_post.j b/src/Applications/GEOSctm_App/ctm_post.j index ead897e..43fb2cf 100755 --- a/src/Applications/GEOSctm_App/ctm_post.j +++ b/src/Applications/GEOSctm_App/ctm_post.j @@ -4,12 +4,13 @@ # Batch Parameters for Post-Processing Job ####################################################################### -#PBS -l walltime=@POST_T +#@BATCH_TIME@POST_T #@POST_P -#PBS -N @POST_N +#@BATCH_JOBNAME@POST_N_@COLLECTION.@YYYYMM #@POST_Q #@BATCH_GROUP -#PBS -o @POST_O +#@BATCH_OUTPUTNAME@POST_O +#@BATCH_JOINOUTERR ####################################################################### # System Environment Variables @@ -32,16 +33,19 @@ setenv GEOSBIN @GEOSBIN setenv GEOSUTIL @GEOSSRC setenv BATCHNAME "@POST_N" -if( $?PBS_NODEFILE ) then - setenv RUN_CMD "@RUN_CMD" +source $GEOSBIN/g5_modules +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib + +if( $?SLURM_NTASKS ) then + setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " + set NCPUS = $SLURM_NTASKS +else if( $?PBS_NODEFILE ) then + setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " set NCPUS = `cat $PBS_NODEFILE | wc -l` else set NCPUS = NULL endif -source $GEOSBIN/g5_modules -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib - ####################################################################### # Perform Post Processing ####################################################################### diff --git a/src/Applications/GEOSctm_App/ctm_quickplot.csh b/src/Applications/GEOSctm_App/ctm_quickplot.csh index 93e80aa..d0dd2a7 100755 --- a/src/Applications/GEOSctm_App/ctm_quickplot.csh +++ b/src/Applications/GEOSctm_App/ctm_quickplot.csh @@ -8,8 +8,16 @@ set LISTFILE = $2 ####################################################################### while( -e $LISTFILE ) + if( -e LOCKFILE ) /bin/echo "Waiting for LOCKFILE ... " + if( -e LOCKFILE ) set LOCK = `stat -c %Z LOCKFILE` + while( -e LOCKFILE ) + set EPOCH = `date +'%s'` + @ LOCKAGE = $EPOCH - $LOCK + if( $LOCKAGE > 120 ) /bin/rm -f LOCKFILE + end lockfile -${JOBID} LOCKFILE + if( -e $LISTFILE ) then set PLOT_COMMAND = `head -1 $LISTFILE` sed 1,1d -i $LISTFILE @@ -21,5 +29,6 @@ while( -e $LISTFILE ) else /bin/rm -f LOCKFILE endif + end diff --git a/src/Applications/GEOSctm_App/ctm_regress.j b/src/Applications/GEOSctm_App/ctm_regress.j index 386d8d4..917205d 100755 --- a/src/Applications/GEOSctm_App/ctm_regress.j +++ b/src/Applications/GEOSctm_App/ctm_regress.j @@ -9,9 +9,9 @@ # Batch Parameters for Regress Job ####################################################################### -#PBS -l walltime=@RUN_T +#@BATCH_TIME@RUN_T #@RUN_P -#PBS -N @REGRESS_N +#@BATCH_JOBNAME@REGRESS_N #@RUN_Q #@BATCH_GROUP @@ -25,6 +25,10 @@ limit stacksize unlimited @SETENVS +# Set OMP_NUM_THREADS +# ------------------- +setenv OMP_NUM_THREADS 1 + @GPUSTART ####################################################################### @@ -34,11 +38,13 @@ limit stacksize unlimited setenv ARCH `uname` setenv SITE @SITE +setenv GEOSDIR @GEOSDIR setenv GEOSBIN @GEOSBIN -setenv RUN_CMD "@RUN_CMD" source $GEOSBIN/g5_modules -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib +setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib + +setenv RUN_CMD "$GEOSBIN/esma_mpirun -np " ####################################################################### # Experiment Specific Environment Variables @@ -63,13 +69,17 @@ cd $HOMDIR set files = `ls -1 *.rc` foreach file ($files) set fname = `echo $file | cut -d "." -f1` - /bin/cp $fname.rc $EXPDIR/regress + @CPEXEC $fname.rc $EXPDIR/regress end cd $EXPDIR/regress /bin/ln -s $EXPDIR/RC/*.rc $EXPDIR/regress -/bin/cp $EXPDIR/GEOSctm.x $EXPDIR/regress -/bin/cp $EXPDIR/linkbcs $EXPDIR/regress +@CPEXEC $EXPDIR/GEOSctm.x $EXPDIR/regress +@CPEXEC $EXPDIR/linkbcs $EXPDIR/regress +@CPEXEC $HOMDIR/*.yaml $EXPDIR/regress +>>>COUPLED<<<@CPEXEC $HOMDIR/*.nml $EXPDIR/regress +>>>MOM6<<<@CPEXEC $HOMDIR/MOM_input $EXPDIR/regress +>>>MOM6<<<@CPEXEC $HOMDIR/MOM_override $EXPDIR/regress cat fvcore_layout.rc >> input.nml @@ -79,8 +89,9 @@ cat $extdata_files > ExtData.rc # Define Atmospheric Resolution # ----------------------------- -set IM = `grep AGCM_IM: $HOMDIR/AGCM.rc | cut -d':' -f2` -set JM = `grep AGCM_JM: $HOMDIR/AGCM.rc | cut -d':' -f2` +set IM = `grep GEOSctm_IM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` +set JM = `grep GEOSctm_JM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` +set LM = `grep GEOSctm_LM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` @ IM6 = 6 * $IM if( $IM6 == $JM ) then @@ -91,29 +102,65 @@ endif # Create Restart List # ------------------- -set rst_types = `cat AGCM.rc | grep "RESTART_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` -set chk_types = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f1 | cut -d "_" -f1-2` -set rst_files = `cat AGCM.rc | grep "RESTART_FILE" | cut -d ":" -f2` -set chk_files = `cat AGCM.rc | grep "CHECKPOINT_FILE" | cut -d ":" -f2` +set rst_files = `cat GEOSCTM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set rst_file_names = `cat GEOSCTM.rc | grep "RESTART_FILE" | grep -v VEGDYN | grep -v "#" | cut -d ":" -f2` + +set chk_files = `cat GEOSCTM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` +set chk_file_names = `cat GEOSCTM.rc | grep "CHECKPOINT_FILE" | grep -v "#" | cut -d ":" -f2` # Remove possible bootstrap parameters (+/-) # ------------------------------------------ -set dummy = `echo $rst_files` -set rst_files = '' +set dummy = `echo $rst_file_names` +set rst_file_names = '' foreach rst ( $dummy ) set length = `echo $rst | awk '{print length($0)}'` set bit = `echo $rst | cut -c1` if( "$bit" == "+" | \ "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length` - set rst_files = `echo $rst_files $rst` + set rst_file_names = `echo $rst_file_names $rst` end # Copy Restarts to Regress directory # ---------------------------------- -foreach rst ( $rst_files ) - /bin/cp $EXPDIR/$rst $EXPDIR/regress +foreach rst ( $rst_file_names ) + @CPEXEC $EXPDIR/$rst $EXPDIR/regress end -/bin/cp $EXPDIR/cap_restart $EXPDIR/regress +@CPEXEC $EXPDIR/cap_restart $EXPDIR/regress + +>>>COUPLED<<>>COUPLED<<<@CPEXEC $EXPDIR/RESTART/* INPUT + +setenv YEAR `cat cap_restart | cut -c1-4` +./linkbcs +if(! -e tile.bin) $GEOSBIN/binarytile.x tile.data tile.bin + +####################################################################### +# Split Saltwater Restart if detected +####################################################################### + +if ( (-e $EXPDIR/regress/openwater_internal_rst) && (-e $EXPDIR/regress/seaicethermo_internal_rst)) then + echo "Saltwater internal state is already split, good to go!" +else + if ( -e $EXPDIR/regress/saltwater_internal_rst ) then + + # The splitter script requires an OutData directory + # ------------------------------------------------- + if (! -d OutData ) mkdir -p OutData + + # Run the script + # -------------- + $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $EXPDIR/regress/saltwater_internal_rst + + # Move restarts + # ------------- + /bin/mv OutData/openwater_internal_rst OutData/seaicethermo_internal_rst . + + # Remove OutData + # -------------- + /bin/rmdir OutData + +endif +endif ####################################################################### # Create Simple History for Efficiency @@ -161,7 +208,7 @@ set EXEMPT_files = `echo SOLAR_INTERNAL_CHECKPOINT_FILE \ set EXEMPT_chk = "" foreach file ( ${EXEMPT_files} ) - set file = `cat AGCM.rc | grep "$file" | cut -d ":" -f2` + set file = `cat GEOSCTM.rc | grep "$file" | cut -d ":" -f2` set EXEMPT_chk = `echo ${EXEMPT_chk} $file` end @@ -171,6 +218,57 @@ set date = `cat cap_restart` set nymd0 = $date[1] set nhms0 = $date[2] +# Select proper MERRA-2 GOCART Emission RC Files +# ---------------------------------------------- +setenv EMISSIONS @EMISSIONS +if( @EMISSIONS =~ MERRA2* ) then + set MERRA2_Transition_Date = 20000401 + + if( $nymd0 < ${MERRA2_Transition_Date} ) then + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/@EMISSIONS/19600101-20000331 + else + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/@EMISSIONS/20000401-present + endif + + if( $LM == 72 ) then + @CPEXEC --remove-destination ${MERRA2_EMISSIONS_DIRECTORY}/*.rc . + else + set files = `/bin/ls -1 ${MERRA2_EMISSIONS_DIRECTORY}/*.rc` + foreach file ($files) + /bin/rm -f `basename $file` + /bin/rm -f dummy + @CPEXEC $file dummy + cat dummy | sed -e "s|/L72/|/L${LM}/|g" | sed -e "s|z72|z${LM}|g" > `basename $file` + end + endif +endif + +if(-e ExtData.rc ) /bin/rm -f ExtData.rc +set extdata_files = `/bin/ls -1 *_ExtData.rc` +cat $extdata_files > ExtData.rc + + +# If REPLAY, link necessary forcing files +# --------------------------------------- +set REPLAY_MODE = `grep REPLAY_MODE: GEOSCTM.rc | grep -v '#' | cut -d: -f2` +if( $REPLAY_MODE == 'Exact' | $REPLAY_MODE == 'Regular' ) then + + set ANA_EXPID = `grep REPLAY_ANA_EXPID: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + set ANA_LOCATION = `grep REPLAY_ANA_LOCATION: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + + set REPLAY_FILE = `grep REPLAY_FILE: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE09 = `grep REPLAY_FILE09: GEOSCTM.rc | grep -v '#' | cut -d: -f2` + set REPLAY_FILE_TYPE = `echo $REPLAY_FILE | cut -d"/" -f1 | grep -v %` + set REPLAY_FILE09_TYPE = `echo $REPLAY_FILE09 | cut -d"/" -f1 | grep -v %` + + # Link REPLAY files + # ----------------- + /bin/ln -sf ${ANA_LOCATION}/aod . + /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE_TYPE} . + /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE09_TYPE} . + +endif + ################################################################## ###### ###### Perform Regression Test # 1 @@ -180,12 +278,12 @@ set nhms0 = $date[2] set test_duration = 240000 -/bin/cp CAP.rc CAP.rc.orig -/bin/cp AGCM.rc AGCM.rc.orig -/bin/cp HISTORY.rc0 HISTORY.rc +@CPEXEC CAP.rc CAP.rc.orig +@CPEXEC GEOSCTM.rc GEOSCTM.rc.orig +@CPEXEC HISTORY.rc0 HISTORY.rc -set NX0 = `grep "^ *NX:" AGCM.rc.orig | cut -d':' -f2` -set NY0 = `grep "^ *NY:" AGCM.rc.orig | cut -d':' -f2` +set NX0 = `grep "^ *NX:" GEOSCTM.rc.orig | cut -d':' -f2` +set NY0 = `grep "^ *NY:" GEOSCTM.rc.orig | cut -d':' -f2` @ NPES0 = $NX0 * $NY0 @@ -196,12 +294,8 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -setenv YEAR `cat cap_restart | cut -c1-4` -./linkbcs -if(! -e tile.bin) $GEOSBIN/binarytile.x tile.data tile.bin - -set NX = `grep "^ *NX": AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` +set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` +set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x @@ -210,9 +304,10 @@ set date = `cat cap_restart` set nymde = $date[1] set nhmse = $date[2] -foreach chk ( $chk_files ) +foreach chk ( $chk_file_names ) /bin/mv $chk ${chk}.${nymde}_${nhmse}.1 end +>>>MOM6<<>>DATAOCEAN<<>>DATAOCEAN<<< @ test_NX = $NPES0 / 6 +>>>DATAOCEAN<<< @ test_NP = $IM / $test_NX +>>>DATAOCEAN<<< if($test_NP < 4 ) then +>>>DATAOCEAN<<< @ test_NX = $IM / 4 # To ensure enough gridpoints for HALO +>>>DATAOCEAN<<< endif +>>>DATAOCEAN<<< set test_NY = 6 +>>>DATAOCEAN<<>>DATAOCEAN<<< set test_NX = $NY0 +>>>DATAOCEAN<<< set test_NY = $NX0 +>>>DATAOCEAN<<>>COUPLED<<>>COUPLED<< cap_restart -/bin/cp CAP.rc.orig CAP.rc -/bin/cp AGCM.rc.orig AGCM.rc -/bin/cp HISTORY.rc0 HISTORY.rc +@CPEXEC CAP.rc.orig CAP.rc +@CPEXEC GEOSCTM.rc.orig GEOSCTM.rc +@CPEXEC HISTORY.rc0 HISTORY.rc ./strip CAP.rc set oldstring = `cat CAP.rc | grep JOB_SGMT:` @@ -247,56 +345,58 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -./strip AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NX:"` +./strip GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NX:"` set newstring = "NX: ${test_NX}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NY:"` +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc setenv YEAR `cat cap_restart | cut -c1-4` ./linkbcs -set NX = `grep "^ *NX": AGCM.rc | cut -d':' -f2` -set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2` +set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` +set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x -foreach rst ( $rst_files ) +foreach rst ( $rst_file_names ) /bin/rm -f $rst end -set numrst = `echo $rst_types | wc -w` -set numchk = `echo $chk_types | wc -w` +set numrst = `echo $rst_files | wc -w` +set numchk = `echo $chk_files | wc -w` @ n = 1 @ z = $numrst + 1 while ( $n <= $numchk ) @ m = 1 while ( $m <= $numrst ) - if( $chk_types[$n] == $rst_types[$m] || \ - \#$chk_types[$n] == $rst_types[$m] ) then - /bin/mv $chk_files[$n] $rst_files[$m] + if( $chk_files[$n] == $rst_files[$m] || \ + \#$chk_files[$n] == $rst_files[$m] ) then + /bin/mv $chk_file_names[$n] $rst_file_names[$m] @ m = $numrst + 999 else @ m = $m + 1 endif end if( $m == $z ) then - echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_types[$n] + echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_files[$n] exit endif @ n = $n + 1 end +>>>COUPLED<<<@CPEXEC RESTART/* INPUT + ################################################################## ###### ###### Perform Regression Test # 3 ###### ################################################################## -set test_duration = 030000 +set test_duration = 060000 if( $CUBE == TRUE ) then set test_NX = 1 @@ -314,20 +414,31 @@ set newstring = "JOB_SGMT: 00000000 ${test_duration}" /bin/mv CAP.rc CAP.tmp cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc -./strip AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NX:"` +./strip GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NX:"` set newstring = "NX: ${test_NX}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc -set oldstring = `cat AGCM.rc | grep "^ *NY:"` +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc +set oldstring = `cat GEOSCTM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" -/bin/mv AGCM.rc AGCM.tmp -cat AGCM.tmp | sed -e "s?$oldstring?$newstring?g" > AGCM.rc +/bin/mv GEOSCTM.rc GEOSCTM.tmp +cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< GEOSCTM.rc +>>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< GEOSCTM.rc + +>>>MOM5<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<< set check = true +>>>MOM6<<< if( $check == true ) then +>>>MOM6<<< echo Comparing "MOM6 restarts" +>>>MOM6<<< cmp $file1 $file2 +>>>MOM6<<< if( $status == 0 ) then +>>>MOM6<<< echo Success! +>>>MOM6<<< echo " " +>>>MOM6<<< else +>>>MOM6<<< echo Failed! +>>>MOM6<<< echo " " +>>>MOM6<<< set pass = false +>>>MOM6<<< endif +>>>MOM6<<< endif +>>>MOM6<< $NCPUS ) then echo "CPU Resources are Over-Specified" echo "--------------------------------" echo "Allotted NCPUs: $NCPUS" - echo "Specified NX : $NX" - echo "Specified NY : $NY" + echo "Requested NCPUs: $NPES" + echo "" + echo "Specified NX: $NX" + echo "Specified NY: $NY" + echo "" + echo "Specified IOSERVER_NODES: $CTM_IOS_NODES" + echo "Specified cores per node: $NCPUS_PER_NODE" exit endif + + else + + @ NPES = $MODEL_NPES + + if( $NPES > $NCPUS ) then + echo "CPU Resources are Over-Specified" + echo "--------------------------------" + echo "Allotted NCPUs: $NCPUS" + echo "Requested NCPUs: $NPES" + echo "" + echo "Specified NX: $NX" + echo "Specified NY: $NY" + exit endif + endif +else + # This is for the desktop path + + @ NPES = $MODEL_NPES + +endif + ####################################################################### -# GCMEMIP Setup +# CTMEMIP Setup ####################################################################### -if( $GCMEMIP == TRUE & ! -e $EXPDIR/restarts/$RSTDATE/cap_restart ) then +if( $CTMEMIP == TRUE & ! -e $EXPDIR/restarts/$RSTDATE/cap_restart ) then cd $EXPDIR/restarts/$RSTDATE @@ -133,21 +188,26 @@ chmod +x $FILE set year = `echo $RSTDATE | cut -d_ -f1 | cut -b1-4` set month = `echo $RSTDATE | cut -d_ -f1 | cut -b5-6` -# Copy MERRA-2 Restarts -# --------------------- -@CPEXEC /discover/nobackup/projects/gmao/g6dev/ltakacs/MERRA2/restarts/AMIP/M${month}/restarts.${year}${month}.tar . +>>>EMIP_OLDLAND<<<# Copy MERRA-2 Restarts +>>>EMIP_OLDLAND<<<# --------------------- +>>>EMIP_NEWLAND<<<# Copy Jason-3_4 REPLAY MERRA-2 NewLand Restarts +>>>EMIP_NEWLAND<<<# ---------------------------------------------- +@CPEXEC /discover/nobackup/projects/gmao/g6dev/ltakacs/@EMIP_MERRA2/restarts/AMIP/M${month}/restarts.${year}${month}.tar . @TAREXEC xf restarts.${year}${month}.tar /bin/rm restarts.${year}${month}.tar -/bin/rm MERRA2*bin +>>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<<# Regrid MERRA-2 Restarts +>>>EMIP_OLDLAND<<<# ----------------------- +>>>EMIP_NEWLAND<<<# Regrid Jason-3_4 REPLAY MERRA-2 NewLand Restarts +>>>EMIP_NEWLAND<<<# ------------------------------------------------ +set RSTID = `/bin/ls *catch* | cut -d. -f1` +set day = `/bin/ls *catch* | cut -d. -f3 | cut -b 7-8` +$GEOSBIN/regrid.pl -np -ymd ${year}${month}${day} -hr 21 -grout C${GEOSCTM_IM} -levsout ${GEOSCTM_LM} -outdir . -d . -expid $RSTID -tagin @EMIP_BCS_IN -oceanin e -i -nobkg -lbl -nolcv -tagout @LSMBCS -rs 3 -oceanout @OCEANOUT +>>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<>>EMIP_OLDLAND<<<$GEOSBIN/gogo.x -s $RSTID.Chem_Registry.rc.${year}${month}${day}_21z -t $EXPDIR/RC/Chem_Registry.rc -i gocart_internal_rst.merra2 -o gocart_internal_rst -r C${GEOSCTM_IM} -l ${GEOSCTM_LM} # Create CAP.rc and cap_restart # ----------------------------- -set nymd = ${year}${month}10 +set nymd = ${year}${month}${day} set nhms = 210000 echo $nymd $nhms > cap_restart @@ -207,11 +271,12 @@ else endif @CPEXEC -f $HOMDIR/*.rc . @CPEXEC -f $HOMDIR/*.nml . + @CPEXEC -f $HOMDIR/*.yaml . @CPEXEC $GEOSBIN/bundleParser.py . cat fvcore_layout.rc >> input.nml -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then @CPEXEC -f $EXPDIR/restarts/$RSTDATE/cap_restart . @CPEXEC -f $EXPDIR/restarts/$RSTDATE/CAP.rc . endif @@ -348,7 +413,7 @@ end # Copy Restarts to Scratch Directory # ---------------------------------- -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then foreach rst ( $rst_file_names ) if(-e $EXPDIR/restarts/$RSTDATE/$rst ) @CPEXEC $EXPDIR/restarts/$RSTDATE/$rst . & end @@ -409,7 +474,7 @@ while ( $counter <= ${NUM_SGMT} ) /bin/rm -f EGRESS -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then @CPEXEC -f $EXPDIR/restarts/$RSTDATE/CAP.rc . else @CPEXEC -f $HOMDIR/CAP.rc . @@ -461,6 +526,19 @@ endif set yearc = `echo $nymdc | cut -c1-4` set yearf = `echo $nymdf | cut -c1-4` +# For Non-Reynolds SST, Modify local CAP.rc Ending date if Finish time exceeds Current year boundary +# -------------------------------------------------------------------------------------------------- +if( @OCEANtag != DE0360xPE0180 ) then + if( $yearf > $yearc ) then + @ yearf = $yearc + 1 + @ nymdf = $yearf * 10000 + 0101 + set oldstring = `cat CAP.rc | grep END_DATE:` + set newstring = "END_DATE: $nymdf $nhmsf" + /bin/mv CAP.rc CAP.tmp + cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc + endif +endif + # Select proper MERRA-2 GOCART Emission RC Files # (NOTE: MERRA2-DD has same transition date) # ---------------------------------------------- @@ -469,7 +547,7 @@ if( ${EMISSIONS} == MERRA2 | \ set MERRA2_Transition_Date = 20000401 if( $nymdc < ${MERRA2_Transition_Date} ) then - set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/$ARCH/etc/$EMISSIONS/19600101-20000331 + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/$EMISSIONS/19600101-20000331 if( $nymdf > ${MERRA2_Transition_Date} ) then set nymdf = ${MERRA2_Transition_Date} set oldstring = `cat CAP.rc | grep END_DATE:` @@ -478,7 +556,7 @@ if( ${EMISSIONS} == MERRA2 | \ cat CAP.tmp | sed -e "s?$oldstring?$newstring?g" > CAP.rc endif else - set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/$ARCH/etc/$EMISSIONS/20000401-present + set MERRA2_EMISSIONS_DIRECTORY = $GEOSDIR/etc/$EMISSIONS/20000401-present endif if( $GEOSCTM_LM == 72 ) then @@ -883,6 +961,13 @@ endif sed -i 's/QFED\/NRT/QFED/' ${COMPNAME}_ExtData_${sYear}.rc sed -i 's/v2.5r1_0.1_deg/v2.5r1\/0.1/' ${COMPNAME}_ExtData_${sYear}.rc +# Environment variables for MPI, etc +# ---------------------------------- + +@SETENVS + +@GPUSTART + # Run bundleParser.py #--------------------- python bundleParser.py @@ -894,10 +979,17 @@ setenv YEAR $yearc # Run GEOSctm.x # ------------- -if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh - @ NPES = $NX * $NY -$RUN_CMD $NPES ./GEOSctm.x -if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh +if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null + +if( $USE_IOSERVER == 1) then + set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $IOS_NODES" +else + set IOSERVER_OPTIONS = "" +endif + +$RUN_CMD $NPES ./GEOSctm.x $IOSERVER_OPTIONS --logging_config 'logging.yaml' + +if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null set rc = $status @@ -1016,7 +1108,7 @@ end # Re-Submit Job ####################################################################### -if( $GCMEMIP == TRUE ) then +if( $CTMEMIP == TRUE ) then foreach rst ( `/bin/ls -1 *_rst` ) /bin/rm -f $EXPDIR/restarts/$RSTDATE/$rst end @@ -1041,9 +1133,9 @@ endif if ( $rc == 0 ) then cd $HOMDIR - if( $GCMEMIP == TRUE ) then - if( $capdate < $enddate ) qsub $HOMDIR/ctm_run.j$RSTDATE + if( $CTMEMIP == TRUE ) then + if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j$RSTDATE else - if( $capdate < $enddate ) qsub $HOMDIR/ctm_run.j + if( $capdate < $enddate ) @BATCH_CMD $HOMDIR/ctm_run.j endif endif diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index ab7acf7..4b263de 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -59,9 +59,9 @@ set BINDIR = `dirname $CTMSETUP` set GEOSDEF = `dirname $BINDIR` set ETCDIR = ${GEOSDEF}/etc -# Test if GEOSgcm.x is here which means you are in install directory +# Test if GEOSctm.x is here which means you are in install directory if (! -x GEOSctm.x) then - echo "You are trying to run $0 in the Applications/GEOSgcm_App directory" + echo "You are trying to run $0 in the Applications/GEOSctm_App directory" echo "This is no longer supported. Please run from the bin/ directory" echo "in your installation" exit 1 @@ -74,6 +74,7 @@ endif # Set default behavior of switches set NOCVS = TRUE set GPU = FALSE +set LINKX = FALSE while ( $#argv > 0 ) set arg = $argv[1] @@ -97,6 +98,11 @@ while ( $#argv > 0 ) set NOCVS = TRUE breaksw + # Symlink GEOSctm.x + case --link: + set LINKX = TRUE + breaksw + # Here any string not above will trigger USAGE case -[Hh]: case --[Hh][Ee][Ll][Pp]: @@ -110,6 +116,7 @@ end ####################################################################### setenv NODE `uname -n` +setenv ARCH `uname -s` setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc` if ( $SITE != 'NCCS' && $SITE != 'NAS' ) then @@ -136,7 +143,9 @@ if ( $SITE == 'NCCS' ) then endif else if ( $SITE == 'NAS' ) then - if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || ($NODE =~ maia*) ) then + if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || \ + ($NODE =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*) || \ + ($NODE =~ maia*) ) then goto ONCOMPUTE endif endif @@ -149,19 +158,12 @@ endif setenv BASEDIR `awk '{print $2}' $ETCDIR/BASEDIR.rc` -if ( `echo $BASEDIR | grep -i ifort` != '') then - set COMPILER = ifort -else if ( `echo $BASEDIR | grep -i pgfortran` != '') then - set COMPILER = pgfortran -else - # Assume default is ifort in case of older baselibs - set COMPILER = ifort -endif - if ( `echo $BASEDIR | grep -i mvapich2` != '') then set MPI = mvapich2 else if ( `echo $BASEDIR | grep -i openmpi` != '') then set MPI = openmpi +else if ( `echo $BASEDIR | grep -i hpcx` != '') then + set MPI = openmpi else if ( `echo $BASEDIR | grep -i impi` != '') then set MPI = intelmpi else if ( `echo $BASEDIR | grep -i intelmpi` != '') then @@ -173,24 +175,6 @@ else set MPI = intelmpi endif -####################################################################### -# Set up RUN_CMD for MPI Stack -####################################################################### - -if ( $MPI == mvapich2 ) then - if ( $SITE == 'NAS') then - setenv RUN_CMD 'mpirun_rsh -export -hostfile $PBS_NODEFILE -np ' # mpi run for MVAPICH2 - else if ($SITE == 'NCCS') then - setenv RUN_CMD 'srun --mpi=pmi2 -n ' # mpi run for MVAPICH2 - endif -else if ( $MPI == openmpi ) then - setenv RUN_CMD 'mpirun -map-by core -bind-to core -np ' # mpi run for OpenMPI -else if ( $MPI == intelmpi ) then - setenv RUN_CMD 'mpirun -np ' # mpi run for Intel MPI -else if ( $MPI == mpt ) then - setenv RUN_CMD 'mpiexec_mpt -np ' # mpi run for MPT -endif - ####################################################################### # Enter Experiment Specific Run Parameters ####################################################################### @@ -249,61 +233,227 @@ endif HRCODE: echo "Enter the ${C1}Atmospheric Horizontal Resolution${CN} code:" -echo "-----------------------------------------------------------" -echo " Cubed-Sphere " -echo "-----------------------------------------------------------" -echo " ${C2}c48 -- 2 deg ${CN}" -echo " ${C2}c90 -- 1 deg ${CN}" -echo " ${C2}c180 -- 1/2 deg (${C1}56-km${C2}) ${CN}" -echo " ${C2}c360 -- 1/4 deg (${C1}28-km${C2}) ${CN} " -echo " ${C2}c720 -- 1/8 deg (${C1}14-km${C2}) ${CN}" -echo " ${C2}c1440 -- 1/16 deg (${C1} 7-km${C2}) ${CN}" +echo "--------------------------------------" +echo " Cubed-Sphere" +echo "--------------------------------------" +echo " ${C2}c12 -- 8 deg ${CN}" +echo " ${C2}c24 -- 4 deg ${CN}" +echo " ${C2}c48 -- 2 deg ${CN}" +echo " ${C2}c90 -- 1 deg ${CN}" +echo " ${C2}c180 -- 1/2 deg (${C1}56-km${C2}) ${CN}" +echo " ${C2}c360 -- 1/4 deg (${C1}28-km${C2}) ${CN} " +echo " ${C2}c720 -- 1/8 deg (${C1}14-km${C2}) ${CN}" +echo " ${C2}c1440 - 1/16 deg (${C1} 7-km${C2}) ${CN}" +echo " ${C2} DYAMOND Grids " +echo " ${C2}c768 -- 1/8 deg (${C1}12-km${C2}) ${CN}" +echo " ${C2}c1536 - 1/16 deg (${C1} 6-km${C2}) ${CN}" +echo " ${C2}c3072 - 1/32 deg (${C1} 3-km${C2}) ${CN}" echo " " set HRCODE = `echo $<` set HRCODE = `echo $HRCODE | tr "[:upper:]" "[:lower:]"` -if( $HRCODE != 'c24' & \ +if( $HRCODE != 'c12' & \ + $HRCODE != 'c24' & \ $HRCODE != 'c48' & \ $HRCODE != 'c90' & \ $HRCODE != 'c180' & \ $HRCODE != 'c360' & \ $HRCODE != 'c720' & \ - $HRCODE != 'c1440' ) goto HRCODE + $HRCODE != 'c1440' & \ + $HRCODE != 'c768' & \ + $HRCODE != 'c1536' & \ + $HRCODE != 'c3072' ) goto HRCODE + +set Resolution = $HRCODE + +set CUBED = TRUE +set DYCORE = FV3 +set GEOSCTM_NF = 6 +set GRID_TYPE = "Cubed-Sphere" +if (`echo $Resolution[1] | cut -b1` == "c" ) then + set GEOSCTM_IM = $Resolution[1] +else + set GEOSCTM_IM = "c$Resolution[1]" +endif + +echo "Enter the Atmospheric Model ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)" +set GEOSCTM_LM = $< +if( .$GEOSCTM_LM == . ) then + set GEOSCTM_LM = 72 +endif + +####################################################################### +# Test to see if using hydrostatic or non-hydrostatic atmosphere +####################################################################### + +ASKHYDRO: + +set DEFAULT_HYDROSTATIC = @CFG_HYDROSTATIC@ + +echo "Use ${C1}Hydrostatic Atmosphere${CN}? (Default: ${C2}${DEFAULT_HYDROSTATIC}${CN})" +set USE_HYDROSTATIC = $< -if( $HRCODE == 'c24' | \ - $HRCODE == 'c48' | \ - $HRCODE == 'c90' | \ - $HRCODE == 'c180' | \ +if( .$USE_HYDROSTATIC == . ) then + set USE_HYDROSTATIC = $DEFAULT_HYDROSTATIC +else + set USE_HYDROSTATIC = `echo $USE_HYDROSTATIC | tr "[:lower:]" "[:upper:]"` + if( $USE_HYDROSTATIC == "Y" | \ + $USE_HYDROSTATIC == "YES" | \ + $USE_HYDROSTATIC == "T" | \ + $USE_HYDROSTATIC == "TRUE" ) set USE_HYDROSTATIC = TRUE + if( $USE_HYDROSTATIC == "N" | \ + $USE_HYDROSTATIC == "NO" | \ + $USE_HYDROSTATIC == "F" | \ + $USE_HYDROSTATIC == "FALSE" ) set USE_HYDROSTATIC = FALSE + + if( "$USE_HYDROSTATIC" != "TRUE" & "$USE_HYDROSTATIC" != "FALSE" ) then + echo + echo "Use ${C1}Hydrostatic Atmosphere${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!" + goto ASKHYDRO + endif +endif + + +####################################################################### +# Test to see if you want to use ioserver +####################################################################### + +ASKIOS: + +if( $HRCODE == 'c180' | \ $HRCODE == 'c360' | \ $HRCODE == 'c720' | \ - $HRCODE == 'c1440' ) then - set Resolution = $HRCODE + $HRCODE == 'c1440' | \ + $HRCODE == 'c768' | \ + $HRCODE == 'c1536' | \ + $HRCODE == 'c3072' ) then + + set DEFAULT_DO_IOS = TRUE + echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}YES${CN} or ${C2}TRUE${CN})" +else + set DEFAULT_DO_IOS = FALSE + echo "Do you wish to ${C1}IOSERVER${CN}? (Default: ${C2}NO${CN} or ${C2}FALSE${CN})" endif -if( $#Resolution == 2 ) then - set GEOSCTM_IM = $Resolution[1] - set GEOSCTM_JM = $Resolution[2] - set CUBED = FALSE - set DYCORE = FV +set DO_IOS = $< +if( .$DO_IOS == . ) then + set DO_IOS = $DEFAULT_DO_IOS else - set CUBED = TRUE - set DYCORE = FV3 - if (`echo $Resolution[1] | cut -b1` == "c" ) then - set GEOSCTM_IM = $Resolution[1] - else - set GEOSCTM_IM = "c$Resolution[1]" + set DO_IOS = `echo $DO_IOS | tr "[:lower:]" "[:upper:]"` + if( $DO_IOS == "Y" | \ + $DO_IOS == "YES" | \ + $DO_IOS == "T" | \ + $DO_IOS == "TRUE" ) set DO_IOS = TRUE + if( $DO_IOS == "N" | \ + $DO_IOS == "NO" | \ + $DO_IOS == "F" | \ + $DO_IOS == "FALSE" ) set DO_IOS = FALSE + + if( $DO_IOS != "TRUE" & $DO_IOS != "FALSE" ) then + echo + echo "${C1}IOSERVER${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!" + goto ASKIOS endif endif +if ( $DO_IOS == "TRUE" ) then + set USE_IOSERVER = 1 +else + set USE_IOSERVER = 0 +endif -echo "Enter the Atmospheric Model ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)" -set GEOSCTM_LM = $< -if( .$GEOSCTM_LM == . ) then - set GEOSCTM_LM = 72 +####################################################################### +# What Processor Should We Run On? +####################################################################### + +ASKPROC: + +if ( $SITE == 'NCCS' ) then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}hasw (Haswell)${CN} (default)" + if (-e /etc/os-release) then + echo " ${C2}sky (Skylake)${CN}" + endif + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'hasw' + endif + + if( $MODEL != 'hasw' & \ + $MODEL != 'sky' ) goto ASKPROC + + if ( $MODEL == 'hasw') then + set NCPUS_PER_NODE = 28 + else if ($MODEL == 'sky') then + set NCPUS_PER_NODE = 40 + endif + +else if ( $SITE == 'NAS' ) then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}has (Haswell)${CN}" + echo " ${C2}bro (Broadwell)${CN}" + echo " ${C2}sky (Skylake)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" + #echo " ${C2}rom (AMD Rome)${CN}" + echo " " + echo " NOTE: Due to how FV3 is compiled by default, Sandy Bridge" + echo " and Ivy Bridge are not supported by current GEOS" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'sky' + endif + + if( $MODEL != 'has' & \ + $MODEL != 'bro' & \ + $MODEL != 'sky' & \ + $MODEL != 'cas' & \ + $MODEL != 'rom' ) goto ASKPROC + + # Some processors have weird names at NAS + # --------------------------------------- + + if ($MODEL == bro ) then + set MODEL = 'bro_ele' + else if ($MODEL == sky) then + set MODEL = 'sky_ele' + else if ($MODEL == cas) then + set MODEL = 'cas_ait' + else if ($MODEL == rom) then + set MODEL = 'rom_ait' + endif + + if ( $MODEL == 'san') then + set NCPUS_PER_NODE = 16 + else if ($MODEL == 'ivy') then + set NCPUS_PER_NODE = 20 + else if ($MODEL == 'has') then + set NCPUS_PER_NODE = 24 + else if ($MODEL == 'bro_ele') then + set NCPUS_PER_NODE = 28 + else if ($MODEL == 'sky_ele') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'cas_ait') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'rom_ait') then + set NCPUS_PER_NODE = 128 + # Romes are on a different aoe + set MODEL='rom_ait:aoe=sles15' + endif + +else + set MODEL = 'UNKNOWN' endif +####################################################################### +# Check for Ocean +####################################################################### -set BCSTAG = Icarus_Reynolds +# The CTM for now seems to only use Reynolds +set OCEAN_TAG = Reynolds # Set DEFAULT SHMEM Parameter # --------------------------- @@ -312,7 +462,7 @@ set BCSTAG = Icarus_Reynolds # Set IAU-Forcing and Bias Correction OFF # --------------------------------------- set FORCEDAS = "#" - set FORCEGCM = "#" + set FORCECTM = "#" # Set Default Convert Parameters # ------------------------------ @@ -324,6 +474,10 @@ set BCSTAG = Icarus_Reynolds set NUM_READERS = 1 set NUM_WRITERS = 1 +# Set DEFAULT IOSERVER Parameters +# ------------------------------- + set DEF_IOS_NDS = 1 + # Default Run Parameters # ---------------------- if( $GEOSCTM_IM == "c12" ) then @@ -336,13 +490,13 @@ if( $GEOSCTM_IM == "c12" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 2 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0012x6C + set ATMOS_RES = CF0012x6C set POST_NDS = 4 set CNV_NX = 1 set CNV_NY = 6 @@ -357,13 +511,13 @@ if( $GEOSCTM_IM == "c24" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0024x6C + set ATMOS_RES = CF0024x6C set POST_NDS = 4 set CNV_NX = 1 set CNV_NY = 6 @@ -378,13 +532,13 @@ if( $GEOSCTM_IM == "c48" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = 180 set HIST_JM = 91 set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000015 set NUM_SGMT = 20 - set ATMOStag = CF0048x6C + set ATMOS_RES = CF0048x6C set POST_NDS = 4 endif if( $GEOSCTM_IM == "c90" ) then @@ -397,13 +551,13 @@ if( $GEOSCTM_IM == "c90" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 4 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set JOB_SGMT = 00000032 set NUM_SGMT = 4 - set ATMOStag = CF0090x6C + set ATMOS_RES = CF0090x6C set POST_NDS = 8 endif if( $GEOSCTM_IM == "c180" ) then @@ -416,15 +570,16 @@ if( $GEOSCTM_IM == "c180" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 6 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 2 set JOB_SGMT = 00000016 set NUM_SGMT = 1 - set ATMOStag = CF0180x6C + set ATMOS_RES = CF0180x6C set POST_NDS = 8 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c360" ) then set DT = 300 @@ -436,16 +591,17 @@ if( $GEOSCTM_IM == "c360" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 12 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 4 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0360x6C + set ATMOS_RES = CF0360x6C set CNV_NX = 4 set POST_NDS = 12 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c500" ) then set DT = 300 @@ -457,17 +613,18 @@ if( $GEOSCTM_IM == "c500" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 12 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 4 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0500x6C + set ATMOS_RES = CF0500x6C set CNV_NX = 8 set POST_NDS = 12 set USE_SHMEM = 1 + set DEF_IOS_NDS = 2 endif if( $GEOSCTM_IM == "c720" ) then set DT = 225 @@ -479,17 +636,18 @@ if( $GEOSCTM_IM == "c720" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 16 set NY = `expr $NX \* 6` - set HYDROSTATIC = TRUE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 6 set JOB_SGMT = 00000005 set NUM_SGMT = 1 - set ATMOStag = CF0720x6C + set ATMOS_RES = CF0720x6C set CNV_NX = 8 set POST_NDS = 16 set USE_SHMEM = 1 + set DEF_IOS_NDS = 3 endif if( $GEOSCTM_IM == "c1440" ) then set DT = 150 @@ -501,32 +659,169 @@ if( $GEOSCTM_IM == "c1440" ) then set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` set NX = 30 set NY = `expr $NX \* 6` - set HYDROSTATIC = FALSE + set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" set NUM_READERS = 6 set JOB_SGMT = 00000001 set NUM_SGMT = 1 - set ATMOStag = CF1440x6C + set ATMOS_RES = CF1440x6C set CNV_NX = 8 set POST_NDS = 32 set USE_SHMEM = 1 + set DEF_IOS_NDS = 4 +endif +# DYAMOND Resolutions +if( $GEOSCTM_IM == "c768" ) then + set DT = 180 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 768 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 32 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF0768x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 3 +endif +if( $GEOSCTM_IM == "c1536" ) then + set DT = 90 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 1536 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 64 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF1536x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 4 +endif +if( $GEOSCTM_IM == "c3072" ) then + set DT = 45 + set SOLAR_DT = 3600 + set IRRAD_DT = 3600 + set OCEAN_DT = $IRRAD_DT + set CHEM_DT = 900 + set GEOSCTM_IM = 3072 + set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` + set NX = 64 + set NY = `expr $NX \* 6` + set HYDROSTATIC = $USE_HYDROSTATIC + set HIST_IM = `expr $GEOSCTM_IM \* 4` + set HIST_JM = `expr $GEOSCTM_IM \* 2 + 1` + set GRID_FILE = "Gnomonic_c${GEOSCTM_IM}.dat" + set NUM_READERS = 6 + set JOB_SGMT = 00000005 + set NUM_SGMT = 1 + set ATMOS_RES = CF3072x6C + set CNV_NX = 8 + set POST_NDS = 16 + set USE_SHMEM = 1 + set DEF_IOS_NDS = 5 endif set IS_FCST = 0 -if($CUBED == "TRUE") then - set FVCUBED = "" - set FVLATLON = "#DELETE" - set GEOSCTM_GRIDNAME = "PE${GEOSCTM_IM}x${GEOSCTM_JM}-CF" - set RES_DATELINE = '${GEOSCTM_IM}x${GEOSCTM_JM}' -endif +set FVCUBED = "" +set GEOSCTM_GRIDNAME = "PE${GEOSCTM_IM}x${GEOSCTM_JM}-CF" +set RES_DATELINE = '${GEOSCTM_IM}x${GEOSCTM_JM}' set BEG_DATE = '18910301 000000' set END_DATE = '29990302 210000' set JOB_SGMT = "$JOB_SGMT 000000" + +####################################################################### +# Setting for Microphysics & Hydrostatic Mode # +####################################################################### + +# Settings for fvcore_layout.rc +set FV_NWAT = "" +set FV_ZTRACER = "z_tracer = .T." +if ( "$USE_HYDROSTATIC" == "TRUE" ) then + set FV_MAKENH = "Make_NH = .F." + set FV_HYDRO = "hydrostatic = .T." + set FV_SATADJ = "do_sat_adj = .F." +else + # Logic for NH runs based on build type + if ( "$DEFAULT_HYDROSTATIC" == "TRUE") then + # If you built for hydrostatic, but want an NH run, run make_nh + set FV_MAKENH = "Make_NH = .T." + else + # If you built for non-hydrostatic and want an NH run, do not run + # NOTE: If you run regrid.pl it will make restarts that should trigger + # make_nh automatically in FV3 + set FV_MAKENH = "Make_NH = .F." + endif + set FV_HYDRO = "hydrostatic = .F." + set FV_SATADJ = "do_sat_adj = .F." + set FV_NWAT = "nwat = 3 " +endif + + +####################################################################### +# Check for land surface and runoff routing models # +####################################################################### + +# NOTE: The CTM uses old-land for now, so hard-coding that in + +# Check for Old vs New Land Surface Model Boundary Conditions +# ----------------------------------------------------------- +#LSM_BCS: +#echo "Enter the choice of ${C1} Land Surface Boundary Conditions${CN} using: ${C2}1${CN} (Icarus), ${C2}2${CN} (Default: Icarus-NLv3)" +#set LSM_BCS = $< +#if( .$LSM_BCS == . ) set LSM_BCS = 2 +#if( $LSM_BCS != 1 & $LSM_BCS != 2 ) then + #echo + #echo "${C1} Land Surface Boundary Conditions${CN} must be set equal to ${C2}1 (Icarus){CN} or ${C2}2 (Icarus-NLv3)${CN}!" + #goto LSM_BCS +#else + #echo +#endif + +set LSM_BCS = 1 + if( $LSM_BCS == 1 ) then + set LSM_BCS = "Icarus" + set LSM_PARMS = "#DELETE" + set EMIP_BCS_IN = "Ganymed-4_0" + set EMIP_OLDLAND = "" + set EMIP_NEWLAND = "#DELETE" + set EMIP_MERRA2 = "MERRA2" + endif + if( $LSM_BCS == 2 ) then + set LSM_BCS = "Icarus-NLv3" + set LSM_PARMS = "" + set EMIP_BCS_IN = "Icarus-NLv3" + set EMIP_OLDLAND = "#DELETE" + set EMIP_NEWLAND = "" + set EMIP_MERRA2 = "MERRA2_NewLand" + endif + + + #------------------------------------ # Convert DT (in seconds) into HHMMSS #------------------------------------ @@ -731,102 +1026,185 @@ else echo endif endif - ####################################################################### # Architecture Specific Variables ####################################################################### -@ NPES = $NX * $NY +@ MODEL_NPES = $NX * $NY + +if ( $DO_IOS == TRUE ) then + set IOS_NDS = $DEF_IOS_NDS +else + set IOS_NDS = 0 +endif + @ CNV_NPES = $CNV_NX * $CNV_NY # Here we need to convert POST_NDS to total tasks. Using 16 cores # per task as a good default @ POST_NPES = $POST_NDS * 16 - setenv RUN_N `echo $EXPID | cut -b1-11`_RUN # RUN Job Name - setenv RUN_FN `echo $EXPID | cut -b1-11`_FCST # Forecast Job Name - setenv POST_N `echo $EXPID | cut -b1-10`_POST # POST Job Name - setenv PLOT_N `echo $EXPID | cut -b1-11`_PLT # PLOT Job Name - setenv MOVE_N `echo $EXPID | cut -b1-11`_MOVE # MOVE Job Name - setenv ARCHIVE_N `echo $EXPID | cut -b1-10`_ARCH # ARCHIVE Job Name - setenv REGRESS_N `echo $EXPID | cut -b1-10`_RGRS # REGRESS Job Name - setenv CONVERT_N `echo $EXPID | cut -b1-11`_CNV # CONVERT Job Name +# Longer job names are now supported with SLURM and PBS. Limits seem to be 1024 characters with SLURM +# and 230 with PBS. To be safe, we will limit to 200 + + setenv RUN_N `echo $EXPID | cut -b1-200`_RUN # RUN Job Name + setenv RUN_FN `echo $EXPID | cut -b1-200`_FCST # Forecast Job Name + setenv POST_N `echo $EXPID | cut -b1-199`_POST # POST Job Name + setenv PLOT_N `echo $EXPID | cut -b1-200`_PLT # PLOT Job Name + setenv MOVE_N `echo $EXPID | cut -b1-200`_MOVE # MOVE Job Name + setenv ARCHIVE_N `echo $EXPID | cut -b1-199`_ARCH # ARCHIVE Job Name + setenv REGRESS_N `echo $EXPID | cut -b1-199`_RGRS # REGRESS Job Name + setenv CONVERT_N `echo $EXPID | cut -b1-200`_CNV # CONVERT Job Name # Default converter time setenv CONVERT_T "0:15:00" # Wallclock Time for ctm_convert.j if( $SITE == 'NAS' ) then + setenv BATCH_CMD "qsub" # PBS Batch command setenv BATCH_GROUP "PBS -W group_list=" # PBS Syntax for GROUP - setenv RUN_FT "6:00:00" # Wallclock Time - UNUSED - setenv RUN_T "8:00:00" # Wallclock Time for ctm_run.j - setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j - setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j - setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j + setenv BATCH_TIME "PBS -l walltime=" # PBS Syntax for walltime + setenv BATCH_JOBNAME "PBS -N " # PBS Syntax for job name + setenv BATCH_OUTPUTNAME "PBS -o " # PBS Syntax for job output name + setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error + setenv RUN_FT "6:00:00" # Wallclock Time for ctm_forecast.j + setenv RUN_T "8:00:00" # Wallclock Time for ctm_run.j + setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j + setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j + setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j if( $GPU == "TRUE" ) then - set NCPUS = 16 # CPUS per node + set NCPUS_PER_NODE = 16 # CPUS per node set MODEL = "san_gpu" # Model of CPU set QTYPE = "gpu_k40" # Queue to use else - set NCPUS = 28 # CPUS per node - set MODEL = "bro" # Model of CPU set QTYPE = "normal" # Queue to use endif - @ NCUS = `echo "($NPES + $NCPUS - 1)/$NCPUS" | bc` - setenv RUN_Q "PBS -q ${QTYPE}" # batch queue name for ctm_run.j - setenv RUN_P "PBS -l select=${NCUS}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_run.j - setenv RUN_FP "PBS -l select=24:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration - UNUSED - setenv POST_Q "PBS -q normal" # batch queue name for ctm_post.j - setenv PLOT_Q "PBS -q normal" # batch queue name for ctm_plot.j - setenv MOVE_Q "PBS -q normal" # batch queue name for ctm_moveplot.j - setenv ARCHIVE_Q "PBS -q normal" # batch queue name for ctm_archive.j - setenv POST_P "PBS -l select=${POST_NDS}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_post.j - setenv PLOT_P "PBS -l select=2:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_plot.j - setenv ARCHIVE_P "PBS -l select=1:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_archive.j - setenv CONVERT_P "PBS -l select=${CNV_NX}:ncpus=${NCPUS}:mpiprocs=${NCPUS}:model=${MODEL}" # PE Configuration for ctm_convert.j - setenv BCSDIR /nobackup/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG # location of Boundary Conditions + + #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` + @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + + setenv RUN_Q "PBS -q ${QTYPE}" # batch queue name for ctm_run.j + setenv RUN_P "PBS -l select=${NODES}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_run.j + setenv RUN_FP "PBS -l select=24:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_forecast.j + setenv POST_Q "PBS -q normal" # batch queue name for ctm_post.j + setenv PLOT_Q "PBS -q normal" # batch queue name for ctm_plot.j + setenv MOVE_Q "PBS -q normal" # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q "PBS -q normal" # batch queue name for ctm_archive.j + setenv POST_P "PBS -l select=${NPCUS}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_post.j + setenv PLOT_P "PBS -l select=1:ncpus=${NCPUS_PER_NODE}:mpiprocs=1:model=${MODEL}" # PE Configuration for ctm_plot.j + setenv ARCHIVE_P "PBS -l select=1:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_archive.j + setenv CONVERT_P "PBS -l select=${CNV_NX}:ncpus=${NCPUS_PER_NODE}:mpiprocs=${NCPUS_PER_NODE}:model=${MODEL}" # PE Configuration for ctm_convert.j + setenv MOVE_P "PBS -l select=1:ncpus=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /nobackup/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID ONLY_MERRA2_SUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION ONLY_MERRA2_SUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION /nobackup/gmao_SIteam/ModelData/verification/MERRA-2 # Default Analysis Location for M2 REPLAY + setenv CHMDIR /nobackup/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /nobackup/$LOGNAME # user work directory - setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing - UNUSED + setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing setenv CPEXEC 'mcp -a' # Copy utility for large copies setenv TAREXEC mtar # Tar utility for large archives else if( $SITE == 'NCCS' ) then - setenv BATCH_GROUP "SBATCH -A " # PBS Syntax for GROUP - setenv RUN_FT "06:00:00" # Wallclock Time - UNUSED + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "2:00:00" # Wallclock Time for ctm_archive.j if( $GPU == "TRUE" ) then - setenv RUN_Q "SBATCH --constraint=k40" # batch queue name for ctm_run.j - else - setenv RUN_Q "SBATCH --constraint=hasw" # batch queue name for ctm_run.j + set NCPUS_PER_NODE = 16 # CPUS per node + set MODEL = "k40" # Model of CPU endif - setenv RUN_P "SBATCH --ntasks=${NPES}" # PE Configuration for ctm_run.j - setenv RUN_FP "SBATCH --ntasks=\&NPES" # PE Configuration - UNUSED - setenv POST_Q "SBATCH --constraint=hasw" # batch queue name for ctm_post.j - setenv PLOT_Q "SBATCH --constraint=hasw" # batch queue name for ctm_plot.j - setenv MOVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_moveplot.j - setenv ARCHIVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_archive.j + + #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` + @ NPCUS = `echo "($POST_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` + + setenv RUN_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_run.j + setenv RUN_P "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_run.j + setenv RUN_FP "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_forecast.j + setenv POST_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_post.j + setenv PLOT_Q "SBATCH --constraint=${MODEL}" # batch queue name for ctm_plot.j + setenv MOVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q "SBATCH --partition=datamove" # batch queue name for ctm_archive.j + setenv POST_P "SBATCH --nodes=${NPCUS} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for ctm_post.j + setenv PLOT_P "SBATCH --nodes=4 --ntasks=4" # PE Configuration for ctm_plot.j + setenv ARCHIVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_archive.j + setenv CONVERT_P "SBATCH --ntasks=${CNV_NPES}" # PE Configuration for ctm_convert.j + setenv MOVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /discover/nobackup/ltakacs/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID x0039 # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/g6dev/ltakacs/x0039 # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION /discover/nobackup/projects/gmao/share/gmao_ops/verification/MERRA-2 # Default Analysis Location for M2 REPLAY + + setenv CHMDIR $SHARE/dao_ops/fvInput_nc3 # locations of Aerosol Chemistry BCs + setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory + setenv COUPLEDIR /discover/nobackup/yvikhlia/coupled/Forcings # Coupled Ocean/Atmos Forcing + setenv CPEXEC /bin/cp # Copy utility for large copies + setenv TAREXEC tar +else if( $SITE == 'AWS' ) then + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP DELETE # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j + setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j + setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j + setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j + setenv ARCHIVE_T "1:00:00" # Wallclock Time for ctm_archive.j + setenv RUN_Q DELETE # batch queue name for ctm_run.j + setenv RUN_P "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for ctm_run.j + setenv RUN_FP "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for ctm_forecast.j + setenv POST_Q NULL # batch queue name for ctm_post.j + setenv PLOT_Q NULL # batch queue name for ctm_plot.j + setenv MOVE_Q NULL # batch queue name for ctm_moveplot.j + setenv ARCHIVE_Q NULL # batch queue name for ctm_archive.j setenv POST_P "SBATCH --ntasks=${POST_NPES}" # PE Configuration for ctm_post.j setenv PLOT_P "SBATCH --nodes=4 --ntasks=4" # PE Configuration for ctm_plot.j setenv ARCHIVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_archive.j setenv CONVERT_P "SBATCH --ntasks=${CNV_NPES}" # PE Configuration for ctm_convert.j - setenv BCSDIR /discover/nobackup/ltakacs/bcs/Icarus/$BCSTAG # location of Boundary Conditions - setenv CHMDIR $SHARE/dao_ops/fvInput_nc3 # locations of Aerosol Chemistry BCs - setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory - setenv COUPLEDIR /discover/nobackup/yvikhlia/coupled/Forcings # Coupled Ocean/Atmos Forcing - UNUSED + setenv MOVE_P "SBATCH --ntasks=1" # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY + + setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs + setenv WRKDIR /home/$LOGNAME # user work directory + setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing + setenv CPEXEC /bin/cp # Copy utility for large copies setenv TAREXEC tar # Tar utility for large archives + # By default on AWS, just ignore IOSERVER for now until testing + set USE_IOSERVER = 0 + set IOS_NDS = 0 + set NCPUS_PER_NODE = 0 else # These are defaults for the desktop - setenv BATCH_GROUP "SBATCH -A " # PBS Syntax for GROUP - setenv RUN_FT "06:00:00" # Wallclock Time - UNUSED + setenv BATCH_CMD "sbatch" # SLURM Batch command + setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name + setenv BATCH_TIME "SBATCH --time=" # SLURM Syntax for walltime + setenv BATCH_JOBNAME "SBATCH --job-name=" # SLURM Syntax for job name + setenv BATCH_OUTPUTNAME "SBATCH --output=" # SLURM Syntax for job output name + setenv BATCH_JOINOUTERR "DELETE" # SLURM joins out and err by default + setenv RUN_FT "06:00:00" # Wallclock Time for ctm_forecast.j setenv RUN_T "12:00:00" # Wallclock Time for ctm_run.j setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "1:00:00" # Wallclock Time for ctm_archive.j setenv RUN_Q NULL # batch queue name for ctm_run.j setenv RUN_P NULL # PE Configuration for ctm_run.j - setenv RUN_FP NULL # PE Configuration - UNUSED + setenv RUN_FP NULL # PE Configuration for ctm_forecast.j setenv POST_Q NULL # batch queue name for ctm_post.j setenv PLOT_Q NULL # batch queue name for ctm_plot.j setenv MOVE_Q NULL # batch queue name for ctm_moveplot.j @@ -835,22 +1213,26 @@ else setenv PLOT_P NULL # PE Configuration for ctm_post.j setenv ARCHIVE_P NULL # PE Configuration for ctm_archive.j setenv CONVERT_P NULL # PE Configuration for ctm_convert.j - setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG # location of Boundary Conditions + setenv MOVE_P NULL # PE Configuration for ctm_moveplot.j + + setenv BCSDIR /ford1/share/gmao_SIteam/ModelData/bcs/${LSM_BCS}/${LSM_BCS}_${OCEAN_TAG} # location of Boundary Conditions + setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY + setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY + setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY + setenv CHMDIR /ford1/share/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /home/$LOGNAME # user work directory - setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing - UNUSED - if( $CUBED == "TRUE" ) then + setenv COUPLEDIR /ford1/share/gmao_SIteam/ModelData/Forcings # Coupled Ocean/Atmos Forcing set NX = 1 set NY = 6 - else - set NX = 2 - set NY = 4 - endif set CNV_NX = ${NX} set CNV_NY = ${NY} - setenv RUN_CMD 'mpirun -oversubscribe -np ' # mpi run for DESKTOP (assumes OpenMPI) setenv CPEXEC /bin/cp # Copy utility for large copies setenv TAREXEC tar # Tar utility for large archives + # By default on desktop, just ignore IOSERVER for now + set USE_IOSERVER = 0 + set IOS_NDS = 0 + set NCPUS_PER_NODE = 0 endif @@ -868,20 +1250,20 @@ while( $check == FALSE ) echo $HISTORYrc > $HOME/.HISTORYrc endif - echo "Enter the tag or directory (/filename) of the ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} to use" + echo "Enter the directory (/filename) of the ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} to use" echo "(To use ${C1}HISTORY.GEOSCTM.rc.tmpl${CN} from current build, Type: ${C2}Current${CN} )" echo "-------------------------------------------------------------------------" - echo "Hit ENTER to use Default Tag/Location: (${C2}${HISTORYrc}${CN})" + echo "Hit ENTER to use Default Location: (${C2}${HISTORYrc}${CN})" set NUHISTORY = $< if( .$NUHISTORY != . ) set HISTORYrc = $NUHISTORY - if( -e $BINDIR/HISTORY.rc.hold ) /bin/mv -f $BINDIR/HISTORY.rc.hold $BINDIR/HISTORY.rc.tmpl - if( -e $BINDIR/HISTORY.rc.tmpl ) /bin/mv -f $BINDIR/HISTORY.rc.tmpl $BINDIR/HISTORY.rc.hold + + set TMPHIST = `mktemp` if( "$HISTORYrc" == "Current" ) then set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $ETCDIR/HISTORY.GEOSCTM.rc.tmpl $BINDIR/HISTORY.rc.tmpl + /bin/cp -f $ETCDIR/HISTORY.GEOSCTM.rc.tmpl $TMPHIST endif if( "$HISTORYrc" != "Current" ) then @@ -889,29 +1271,30 @@ while( $check == FALSE ) set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $ETCDIR/$HISTORYrc $BINDIR/HISTORY.rc.tmp1 + set TMPHIST1 = `mktemp` + /bin/cp -f $ETCDIR/$HISTORYrc $TMPHIST1 - set EXPID_old = `grep "EXPID:" $BINDIR/HISTORY.rc.tmp1 | cut -d: -f2` - set EXPDSC_old = `grep "EXPDSC:" $BINDIR/HISTORY.rc.tmp1 | cut -d: -f2` + set EXPID_old = `grep "EXPID:" $TMPHIST1 | cut -d: -f2` + set EXPDSC_old = `grep "EXPDSC:" $TMPHIST1 | cut -d: -f2` - /bin/rm -f command + set TMPCMD = `mktemp` set string = "EXPID:" - echo cat $BINDIR/HISTORY.rc.tmp1 \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \"${EXPID_old}\" , \"${EXPID}\" \)\;print\} else print\}\' \> $BINDIR/HISTORY.rc.tmpl > command - chmod +x command - ./command - /bin/rm -f command - /bin/mv -f $BINDIR/HISTORY.rc.tmpl $BINDIR/HISTORY.rc.tmp1 - cat $BINDIR/HISTORY.rc.tmp1 | sed -e "s|${EXPDSC_old}|${EXPDSC}|g" > $BINDIR/HISTORY.rc.tmpl - /bin/rm -f $BINDIR/HISTORY.rc.tmp1 + echo cat $TMPHIST1 \| awk \'\{if \( \$1 \~ \"${string}\" \) \ + \{sub \( \"${EXPID_old}\" , \"${EXPID}\" \)\;print\} else print\}\' \> $TMPHIST > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD + /bin/mv -f $TMPHIST $TMPHIST1 + cat $TMPHIST1 | sed -e "s|${EXPDSC_old}|${EXPDSC}|g" > $TMPHIST + /bin/rm -f $TMPHIST1 else if( -e $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl ) then set check = TRUE if( -e $HOME/.HISTORYrc ) /bin/rm $HOME/.HISTORYrc echo $HISTORYrc > $HOME/.HISTORYrc - /bin/cp -f $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl $BINDIR/HISTORY.rc.tmpl + /bin/cp -f $HISTORYrc/HISTORY.GEOSCTM.rc.tmpl $TMPHIST else - echo "This condition is based on updating HISTORY.AGCM.rc.tmpl with CVS" + echo "This condition is based on updating HISTORY.GEOSCTM.rc.tmpl with CVS" echo "This has no equivalent in git at present. Please contact Matt" echo "Thompson or Larry Takacs to help resolve this." exit 2 @@ -1039,7 +1422,8 @@ setenv CNVDIR $EXPDIR/convert ####################################################################### # GEOSctm Build Directory -# ------------------------ +# ----------------------- + echo "Enter Location for ${C1}Build${CN} directory containing: bin/ etc/ include/ etc..." echo "Hit ENTER to use Default Location:" echo "----------------------------------" @@ -1050,8 +1434,9 @@ setenv CNVDIR $EXPDIR/convert if( .$NUGEOSDIR != . ) then set GEOSDIR = $NUGEOSDIR endif - + set check = FALSE + GEOSDIR: while( $check == FALSE ) @@ -1081,9 +1466,13 @@ while( $check == FALSE ) endif end +# +# GEOSBIN does point to the bin/ directory in each +# setenv GEOSSRC ${GEOSDIR} setenv GEOSBIN ${GEOSDIR}/bin setenv GEOSETC ${GEOSDIR}/etc + setenv GEOSUTIL ${GEOSSRC} setenv CTMVER `cat ${GEOSETC}/.GEOSCTM_VERSION` @@ -1125,16 +1514,27 @@ echo $GROUP > $HOME/.GROUProot set EMISSIONS = g5chem endif +if ( $LINKX == "TRUE" ) then + if ( -e $GEOSBIN/GEOSctm.x ) ln -s $GEOSBIN/GEOSctm.x $EXPDIR +else if ( -e $GEOSBIN/GEOSctm.x ) rsync -avx $GEOSBIN/GEOSctm.x $EXPDIR +endif ####################################################################### -# Create SETENV Commands +# Set Recommended MPI Stack Settings ####################################################################### +# By default do not write restarts by oserver +set RESTART_BY_OSERVER = NO + /bin/rm -f $HOMDIR/SETENV.commands if( $MPI == openmpi ) then +# Open MPI and GEOS has issues with restart writing. Having the +# oserver write them can be orders of magnitude faster +set RESTART_BY_OSERVER = YES + # This turns off an annoying warning when running # Open MPI on a system where TMPDIRs are on a networked # file system @@ -1178,7 +1578,7 @@ cat > $HOMDIR/SETENV.commands << EOF setenv MPI_COLL_REPRODUCIBLE setenv SLURM_DISTRIBUTION block - + #setenv MPI_DISPLAY_SETTINGS 1 #setenv MPI_VERBOSE 1 @@ -1191,14 +1591,40 @@ cat > $HOMDIR/SETENV.commands << EOF # with binarytile.x and other executables. unsetenv PMI_RANK + # Often when debugging on MPT, the traceback from Intel Fortran + # is "absorbed" and only MPT's errors are displayed. To allow the + # compiler's traceback to be displayed, uncomment this environment + # variable + #setenv FOR_IGNORE_EXCEPTIONS false + EOF else if( $MPI == intelmpi ) then cat > $HOMDIR/SETENV.commands << EOF setenv I_MPI_DAPL_UD enable +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + +# This flag prints out the Intel MPI state. Uncomment if needed +#setenv I_MPI_DEBUG 9 EOF +# These are options determined to be useful at NCCS +# Not setting generally as they are more fabric/cluster +# specific compared to the above adjustments +if ( $SITE == 'NCCS' ) then + +cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_SHM_HEAP_VSIZE 512 +setenv PSM2_MEMORY large +setenv I_MPI_EXTRA_FILESYSTEM 1 +setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs +setenv ROMIO_FSTYPE_FORCE "gpfs:" +EOF + +endif # if NCCS + endif # if mpi ####################################################################### @@ -1315,6 +1741,7 @@ s?@PLOT_T?$PLOT_T?g s?@PLOT_P?$PLOT_P?g s?@PLOT_Q?$PLOT_Q?g s?@MOVE_Q?$MOVE_Q?g +s?@MOVE_P?$MOVE_P?g s?@ARCHIVE_N?$ARCHIVE_N?g s?@ARCHIVE_T?$ARCHIVE_T?g s?@ARCHIVE_P?$ARCHIVE_P?g @@ -1326,27 +1753,37 @@ s?@CONVERT_T?$CONVERT_T?g s?@CNV_NX?$CNV_NX?g s?@CNV_NY?$CNV_NY?g s?@BCSDIR?$BCSDIR?g +s?@LSMBCS?$LSM_BCS?g +s?@EMIP_BCS_IN?$EMIP_BCS_IN?g +s?@EMIP_MERRA2?$EMIP_MERRA2?g +s?@BCSTAG?$OCEAN_TAG?g s?@CHMDIR?$CHMDIR?g s?@EXPDIR?$EXPDIR?g s?@EXPDSC?$EXPDSC?g s?@HOMDIR?$HOMDIR?g s?@CNVDIR?$CNVDIR?g s?@BATCH_GROUP?${BATCH_GROUP}${GROUP}?g +s?@BATCH_TIME?$BATCH_TIME?g +s?@BATCH_CMD?$BATCH_CMD?g +s?@BATCH_JOBNAME?$BATCH_JOBNAME?g +s?@BATCH_OUTPUTNAME?$BATCH_OUTPUTNAME?g +s?@BATCH_JOINOUTERR?$BATCH_JOINOUTERR?g s?@SITE?$SITE?g s?@GEOSDIR?$GEOSDIR?g s?@GEOSSRC?$GEOSSRC?g s?@GEOSBIN?$GEOSBIN?g s?@GEOSETC?$GEOSETC?g s?@GEOSUTIL?$GEOSUTIL?g -s?@RUN_CMD?$RUN_CMD?g s?@CHECKPOINT_TYPE?default?g s?@OBSERVER_FRQ?0?g s?RECORD_?#RECORD_?g +s?@DASTUNING?#?g + s?>>>FORCEDAS<<>>FORCEGCM<<>>FORCECTM<<>>PCHEM<<>>GOCART<<>>GMICHEM<<>>GEOSCHEMCHEM<<>>passiveTracer<<>>IdealizedPT<<>>FVCUBED<<>>FVLATLON<<>>HIST_GOCART<<>>HIST_GMICHEM<<>>HIST_GEOSCHEMCHEM<<>>HIST_passiveTracer<<>>HIST_IdealizedPT<<>>EMIP_OLDLAND<<>>EMIP_NEWLAND<<>>4DIAUDAS<<>>REGULAR_REPLAY<<> $HOMDIR/sedfile << EOF s^@HYDROSTATIC^$HYDROSTATIC^g s/@GRID_FILE/$GRID_FILE/g EOF -endif set FILES = "ctm_run.j \ ctm_post.j \ @@ -1443,6 +1889,7 @@ set FILES = "ctm_run.j \ CAP.rc.tmpl \ GEOSCTM.rc.tmpl \ HISTORY.rc.tmpl \ + logging.yaml \ CTM_GridComp.rc.tmpl \ pTracers_GridComp.rc.tmpl \ FP_ExtData.rc.tmpl \ @@ -1451,22 +1898,22 @@ set FILES = "ctm_run.j \ MERRA2_ExtData.rc.tmpl " set FILES = `echo $FILES` -if($CUBED == "TRUE") then set FILES = "$FILES \ fvcore_layout.rc" set FILES = `echo $FILES` -endif echo " " -# Operate on files in BINDIR +# Operate on files in ETCDIR foreach FILE ($FILES) /bin/rm -f $HOMDIR/tmpfile /bin/rm -f $HOMDIR/$FILE - if ( -e $BINDIR/$FILE ) then + if ( $FILE == "HISTORY.rc.tmpl" ) then + cat $TMPHIST > $HOMDIR/tmpfile + else if ( -e $BINDIR/$FILE ) then cat $BINDIR/$FILE > $HOMDIR/tmpfile else if ( -e $ETCDIR/$FILE ) then cat $ETCDIR/$FILE > $HOMDIR/tmpfile @@ -1490,11 +1937,7 @@ if ($GPU == "TRUE") then /bin/rm -f $HOMDIR/GPUEND.commands endif -if( -e $BINDIR/HISTORY.rc.hold ) then -/bin/mv -f $BINDIR/HISTORY.rc.hold $BINDIR/HISTORY.rc.tmpl -else -/bin/rm -f $BINDIR/HISTORY.rc.tmpl -endif +/bin/rm -f $TMPHIST echo $HOMDIR > $EXPDIR/.HOMDIR echo " " @@ -1520,7 +1963,6 @@ endif set RSNAMES = "LH2O LMAM LCARMA LGMICHEM LGEOSCHEMCHEM LSTRATCHEM LGOCART" set RSTYPES = "INTERNAL IMPORT" -/bin/rm -f command set FILE = GEOSCTM.rc.tmpl if( -e $HOMDIR/$FILE ) set LOCDIR = $HOMDIR @@ -1531,26 +1973,28 @@ foreach rsname ($RSNAMES) set test = `eval echo \$$rsname` if( $test == FALSE ) then foreach type ($RSTYPES) + set TMPCMD = `mktemp` set string = ${name}_${type} /bin/rm -f $LOCDIR/$FILE.tmp /bin/mv -f $LOCDIR/$FILE $LOCDIR/$FILE.tmp echo cat $LOCDIR/$FILE.tmp \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > command - chmod +x command - ./command - /bin/rm -f command + \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD end endif end if( $LGOCART == TRUE ) then + set TMPCMD = `mktemp` /bin/rm -f $LOCDIR/$FILE.tmp /bin/mv -f $LOCDIR/$FILE $LOCDIR/$FILE.tmp set string = AEROCLIM echo cat $LOCDIR/$FILE.tmp \| awk \'\{if \( \$1 \~ \"${string}\" \) \ - \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > command - chmod +x command - ./command - /bin/rm -f command + \{sub \( \/${string}\/ ,\"\#${string}\" \)\;print\} else print\}\' \> $LOCDIR/$FILE > $TMPCMD + chmod +x $TMPCMD + $TMPCMD + /bin/rm -f $TMPCMD endif /bin/rm -f $LOCDIR/$FILE.tmp @@ -1993,7 +2437,7 @@ echo " " echo "Build Directory: ${C2}${GEOSDIR}${CN}" echo "----------------" echo " " -if ( ! -e $GEOSDIR/$ARCH/bin/GEOSctm.x ) then +if ( ! -e $GEOSBIN/GEOSctm.x ) then echo " " echo "Note: Build directory does not contain ${C1}GEOSctm.x${CN} !" echo " You will need to put a copy of ${C1}GEOSctm.x${CN} in your Experiment directory" @@ -2003,7 +2447,7 @@ else echo " " echo "The following executable has been placed in your Experiment Directory:" echo "----------------------------------------------------------------------" - echo "${C2}$GEOSDIR/$ARCH/bin/GEOSctm.x${CN}" + echo "${C2}$GEOSBIN/GEOSctm.x${CN}" echo " " endif echo " " @@ -2268,40 +2712,35 @@ if ( ! -e $HOME/.HOMDIRroot || ! -e $HOME/.EXPDIRroot ) then echo "Please run ctm_setup in non-clone mode at least once to use this script." exit 1 endif -else - # MAT There are two thoughts here. You can either place the clone - # in the .HOMDIRroot/.EXPDIRroot, but that is only nice if you have - # one place, you put all your experiments. What if you have lots of - # directories? Instead, let us default to the directory root of the - # cloned experiment. +endif + +# MAT There are two thoughts here. You can either place the clone in +# the .HOMDIRroot/.EXPDIRroot, but that is only nice if you have one +# single place you put all your experiments. What if you have lots of +# directories? Instead, let us default to the directory root of the +# cloned experiment if the cloned experiment is yours. If it isn't your +# experiment you are cloning, then default to the values in the dotfile - #set HOMDIRroot=`cat $HOME/.HOMDIRroot` +if ( $OLDUSER == $LOGNAME) then set HOMDIRroot=`dirname $OLDHOMDIR` - #echo "Setting HOMDIR to $HOMDIRroot" - #set EXPDIRroot=`cat $HOME/.EXPDIRroot` set EXPDIRroot=`dirname $OLDEXPDIR` - #echo "Setting EXPDIR to $EXPDIRroot" +else + set HOMDIRroot=`cat $HOME/.HOMDIRroot` + set EXPDIRroot=`cat $HOME/.EXPDIRroot` endif +echo "Setting HOMDIR to $HOMDIRroot" +echo "Setting EXPDIR to $EXPDIRroot" + if ( -e $HOME/.GROUProot ) then set GROUProot=`cat $HOME/.GROUProot` - #echo "Using account $GROUProot" + echo "Using account $GROUProot" else echo "$HOME/.GROUProot not found." set GROUProot=`groups | awk '{print $1}'` echo "Based off of groups, setting account to $GROUProot" endif -# ----------------------------------- -# Find out if we are running the cube -# ----------------------------------- - -if ( -e $CLONEDIR/fvcore_layout.rc ) then - set CUBED = TRUE -else - set CUBED = FALSE -endif - # ------------------------------------------------ # Set the new EXPDIR and HOMDIR based on the roots # ------------------------------------------------ @@ -2354,6 +2793,7 @@ HOMDIR/CAP.rc HOMDIR/GEOSCTM.rc HOMDIR/HISTORY.rc HOMDIR/ctm_run.j +HOMDIR/logging.yaml HOMDIR/MAPL_ExtData.rc HOMDIR/CTM_GridComp.rc HOMDIR/pTracers_GridComp.rc @@ -2363,11 +2803,9 @@ HOMDIR/FPIT_ExtData.rc.tmpl HOMDIR/FP_ExtData.rc.tmpl EOF -if($CUBED == "TRUE") then cat >> $FILES_TO_PROCESS << EOF HOMDIR/fvcore_layout.rc EOF -endif # ------------------------------------------------ # Create two sets of files so we can copy from one @@ -2445,14 +2883,14 @@ end # Construct the new job ids # ------------------------- - set RUN_N=`echo $NEWEXPID | cut -b1-11`_RUN - set RUN_FN=`echo $NEWEXPID | cut -b1-11`_FCST - set POST_N=`echo $NEWEXPID | cut -b1-10`_POST - set PLOT_N=`echo $NEWEXPID | cut -b1-11`_PLT - set MOVE_N=`echo $NEWEXPID | cut -b1-11`_MOVE -set ARCHIVE_N=`echo $NEWEXPID | cut -b1-10`_ARCH -set REGRESS_N=`echo $NEWEXPID | cut -b1-10`_RGRS -set CONVERT_N=`echo $NEWEXPID | cut -b1-11`_CNV + set RUN_N=`echo $NEWEXPID | cut -b1-200`_RUN + set RUN_FN=`echo $NEWEXPID | cut -b1-200`_FCST + set POST_N=`echo $NEWEXPID | cut -b1-199`_POST + set PLOT_N=`echo $NEWEXPID | cut -b1-200`_PLT + set MOVE_N=`echo $NEWEXPID | cut -b1-200`_MOVE +set ARCHIVE_N=`echo $NEWEXPID | cut -b1-199`_ARCH +set REGRESS_N=`echo $NEWEXPID | cut -b1-199`_RGRS +set CONVERT_N=`echo $NEWEXPID | cut -b1-200`_CNV sed -i -e "/^#PBS -N/ s#\(PBS -N \)\(.*\)#\1$RUN_N#" \ -e "/^#SBATCH --job-name=/ s#\(SBATCH --job-name=\)\(.*\)#\1$RUN_N#" $NEWHOMDIR/ctm_run.j @@ -2735,7 +3173,6 @@ ctm_setup, a setup script for the GEOS-5 CTM -c --color Set the colors for $0:t -g --gpu Run the model using the GPUs -h --help Show usage - --nocvs Do not archive the source If invoked alone, the script runs as normal. diff --git a/src/Applications/GEOSctm_App/fvcore_layout.rc b/src/Applications/GEOSctm_App/fvcore_layout.rc index cfbab53..72db6c0 100644 --- a/src/Applications/GEOSctm_App/fvcore_layout.rc +++ b/src/Applications/GEOSctm_App/fvcore_layout.rc @@ -1,60 +1,72 @@ &fv_core_nml - fv_debug = .F. - n_sponge = 9 - n_zfilter = 25 - tau = 0. - rf_cutoff = 7.5e2 - d2_bg_k1 = 0.15 - d2_bg_k2 = 0.02 - fv_sg_adj = 450 - remap_option = 0 - kord_tm = 9 - kord_mt = 9 - kord_wz = 9 - kord_tr = 9 - hydrostatic = .T. - make_nh = .T. - beta = 0. - a_imp = 1. - p_fac = 0.1 - k_split = 2 - n_split = 0 - nwat = 1 - nord = 2 - dddmp = 0.1 - d4_bg = 0.12 - d2_bg = 0.0 - d_ext = 0.0 - vtdm4 = 0.0 - delt_max = 0.002 - ke_bg = 0.0 - do_vort_damp = .F. - d_con = 0. - hord_mt = 10 - hord_vt = 10 - hord_tm = 10 - hord_dp = 10 - hord_tr = 8 - adjust_dry_mass = .F. - consv_te = 1. - consv_am = .F. - fill = .T. - dwind_2d = .F. - z_tracer = .F. - / + k_split = 2 + @FV_MAKENH + @FV_HYDRO + @FV_SATADJ + @FV_ZTRACER + @FV_NWAT +/ - &main_nml +&main_nml / - &test_case_nml - test_case = 13 - / +&test_case_nml + test_case = 13 +/ - &fms_io_nml +&fms_io_nml / - &fms_nml - print_memory_usage=.false. - domains_stack_size = 24000000 +&fms_nml + print_memory_usage=.false. + domains_stack_size = 24000000 / +&gfdl_cloud_microphysics_nml + sedi_transport = .true. + do_sedi_heat = .false. + rad_snow = .true. + rad_graupel = .true. + rad_rain = .true. + const_vi = .F. + const_vs = .F. + const_vg = .F. + const_vr = .F. + vi_max = 1. + vs_max = 2. + vg_max = 12. + vr_max = 12. + qi_lim = 1. + prog_ccn = .false. + do_qa = .true. + fast_sat_adj = .true. + tau_l2v = 1200. + tau_v2l = 150. + tau_g2v = 900. + rthresh = 7.e-6 ! This is a key parameter for cloud water + dw_land = 0.16 + dw_ocean = 0.10 + ql_gen = 1.0e-3 + ql_mlt = 1.0e-3 + qi0_crt = 8.0E-5 + qs0_crt = 1.0e-3 + tau_i2s = 1000. + c_psaci = 0.05 + c_pgacs = 0.01 + rh_inc = 0.30 + rh_inr = 0.30 + rh_ins = 0.30 + ccn_l = 300. + ccn_o = 100. + c_paut = 1.0 + c_cracw = 0.8 + use_ppm = .false. + use_ccn = .true. + mono_prof = .true. + z_slope_liq = .true. + z_slope_ice = .true. + de_ice = .false. + fix_negative = .true. + icloud_f = 1 + mp_time = 150. +/ diff --git a/src/Applications/GEOSctm_App/logging.yaml b/src/Applications/GEOSctm_App/logging.yaml new file mode 100644 index 0000000..ff5dc71 --- /dev/null +++ b/src/Applications/GEOSctm_App/logging.yaml @@ -0,0 +1,168 @@ +schema_version: 1 + +############################### +locks: + mpi: + class: MpiLock + comm: MPI_COMM_WORLD + +############################### +formatters: + legacy: + class: Formatter + format: '%(message)a' + + basic: + class: Formatter + format: '%(short_name)a15~: %(level_name)a~: %(message)a' + + mpi: + class: MpiFormatter + format: '%(mpi_rank)i4.4~: %(name)~: %(level_name)a~: %(message)a' + comm: MPI_COMM_WORLD + + column: + class: Formatter + format: '(%(i)i3.3,%(j)i3.3): %(level_name)' + + + +############################### +handlers: + + console: + class: streamhandler +# formatter: legacy + formatter: basic + unit: OUTPUT_UNIT + level: INFO + + + warnings: + class: FileHandler + filename: warnings_and_errors.log + lock: mpi + level: WARNING + formatter: basic + + errors: + class: StreamHandler + formatter: basic + unit: ERROR_UNIT + level: ERROR + + mpi_shared: + class: FileHandler + filename: allPEs.log + formatter: mpi + comm: MPI_COMM_WORLD + lock: mpi + rank_keyword: rank + level: DEBUG + + mpi_debug: + class: MpiFileHandler + formatter: basic + filename: debug_%(rank)i3.3~.log + comm: MPI_COMM_WORLD + rank_prefix: rank + level: DEBUG + +############################### +root: + handlers: [warnings,errors,console] + level: WARNING + root_level: WARNING + +############################### +loggers: + + errors: + handlers: [errors] + level: ERROR + + CAP: + level: WARNING + root_level: INFO + + MAPL: + handlers: [mpi_shared] + level: INFO + root_level: DEBUG + + # Note: When enabling another logger, make sure + # indentation matches that of the above + # loggers! + + # MAPL.base: + # MAPL.generic: + # MAPL.pfio + + # SHARED.GMAOSHARED.GEOSSHARED.QSAT: + + # CAP.GCM.DATATM: + # CAP.GCM.AGCM: + # CAP.GCM.AGCM.SCMDYNAMICS: + # CAP.GCM.AGCM.SCMDYNAMICS.DYN: + # CAP.GCM.AGCM.SCMDYNAMICS.ADV: + # CAP.GCM.AGCM.SUPERDYNAMICS: + # CAP.GCM.AGCM.SUPERDYNAMICS.DYN: + # CAP.GCM.AGCM.SUPERDYNAMICS.ADV: + # CAP.GCM.AGCM.PHYSICS: + # CAP.GCM.AGCM.PHYSICS.GWD: + # CAP.GCM.AGCM.PHYSICS.MOIST: + # CAP.GCM.AGCM.PHYSICS.TURBULENCE: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.HEMCO: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.PCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.ACHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.BC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.BRC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CFC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CH4: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CO2: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.CO: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.DU: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.NI: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.O3: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.OC: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.Rn: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.SS: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.SU: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GOCART.data: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GAAS: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.H2O: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.STRATCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GMICHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.CARMA: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.GEOSCHEMCHEM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.MATRIX: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.MAM: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.TR: + # CAP.GCM.AGCM.PHYSICS.CHEMISTRY.DNA: + # CAP.GCM.AGCM.PHYSICS.SURFACE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.SALTWATER: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAKE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LANDICE: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.VEGDYN: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.SURFACE.LAND.: + # CAP.GCM.AGCM.PHYSICS.RADIATION: + # CAP.GCM.AGCM.PHYSICS.RADIATION.SOLAR: + # CAP.GCM.AGCM.PHYSICS.RADIATION.IRRAD: + # CAP.GCM.AGCM.PHYSICS.RADIATION.SATSIM: + # CAP.GCM.AGCM.ORBIT: + # CAP.GCM.AIAU: + # CAP.GCM.OGCM: + # CAP.GCM.OGCM.OBIO: + # CAP.GCM.OGCM.ORAD: + # CAP.GCM.OGCM.SEAICE: + # CAP.GCM.OGCM.SEAICEdata: + # CAP.GCM.OGCM.OCEAN: + # CAP.GCM.OGCM.OCEAN.DATASEA: + # CAP.GCM.OGCM.OCEAN.MOM: + # CAP.GCM.OGCM.OCEAN.MOM6: + diff --git a/src/Components/GEOSctm_GridComp/.gitignore b/src/Components/GEOSctm_GridComp/.gitignore index 43277ae..2a0075c 100644 --- a/src/Components/GEOSctm_GridComp/.gitignore +++ b/src/Components/GEOSctm_GridComp/.gitignore @@ -1,3 +1,9 @@ /@GEOSchem_GridComp +/GEOSchem_GridComp +/GEOSchem_GridComp@ /@GEOSgcm_GridComp +/GEOSgcm_GridComp +/GEOSgcm_GridComp@ /@FVdycoreCubed_GridComp +/FVdycoreCubed_GridComp +/FVdycoreCubed_GridComp@ diff --git a/src/Components/GEOSctm_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CMakeLists.txt index 5ab47da..07f9a22 100644 --- a/src/Components/GEOSctm_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CMakeLists.txt @@ -1,5 +1,7 @@ esma_set_this () +ecbuild_declare_project() + set (alldirs @GEOSchem_GridComp @FVdycoreCubed_GridComp @@ -12,4 +14,6 @@ set (FV_PRECISION R8) esma_add_library (${this} SRCS GEOS_ctmEnvGridComp.F90 GEOS_ctmGridCompMod.F90 SUBCOMPONENTS ${alldirs} - DEPENDENCIES FVdycoreCubed_GridComp MAPL_Base) + DEPENDENCIES FVdycoreCubed_GridComp MAPL) + +ecbuild_install_project(NAME GEOSctm_GridComp) diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt index bd4fea6..8a2293e 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CMakeLists.txt @@ -1,4 +1,4 @@ -esma_set_this () + esma_set_this () if (Convection_MODE MATCHES stub) set (SRCS @@ -20,6 +20,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL_Base + DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL ) diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 index 5b689af..d542108 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_ConvectionGridCompMod.F90 @@ -13,7 +13,7 @@ MODULE CTM_ConvectionGridCompMod ! !USES: ! USE ESMF - USE MAPL_Mod + USE MAPL USE GmiConvectionMethod_mod ! GMI Convection component USE GenericConvectionMethod_mod ! Generic Convection component USE m_chars, ONLY : uppercase diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 index 96ccfe4..a47a057 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/CTM_rasCalculationsMod.F90 @@ -11,7 +11,7 @@ MODULE CTM_rasCalculationsMod ! !USES: ! use ESMF - use MAPL_Mod + use MAPL use GEOS_Mod use GEOS_UtilsMod diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 index cc3ccab..dc9c646 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GenericConvectionMethod_mod.F90 @@ -13,7 +13,7 @@ module GenericConvectionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL !USE Chem_UtilMod USE convectiveTransport_mod use GmiArrayBundlePointer_mod diff --git a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 index d9d0f31..4649349 100644 --- a/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMconvection_GridComp/GmiConvectionMethod_mod.F90 @@ -13,7 +13,7 @@ module GmiConvectionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL USE Chem_UtilMod use GmiArrayBundlePointer_mod USE GmiESMFrcFileReading_mod diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt index 8bd92fb..603f742 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CMakeLists.txt @@ -18,6 +18,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL_Base + DEPENDENCIES GEOSchem_GridComp Chem_Shared MAPL ) diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 index 750eed4..9e3bf1a 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/CTM_DiffusionGridCompMod.F90 @@ -13,7 +13,7 @@ MODULE CTM_DiffusionGridCompMod ! !USES: ! USE ESMF - USE MAPL_Mod + USE MAPL USE GmiDiffusionMethod_mod ! ESMF parent component USE m_chars, ONLY : uppercase diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 index f10ed4e..8ba41d4 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/GmiDiffusionMethod_mod.F90 @@ -13,7 +13,7 @@ module GmiDiffusionMethod_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL USE Chem_UtilMod USE updateDiffusion_mod use GmiArrayBundlePointer_mod diff --git a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 index 0e01b80..1929231 100644 --- a/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMdiffusion_GridComp/updateDiffusion_mod.F90 @@ -13,7 +13,7 @@ module updateDiffusion_mod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use GmiArrayBundlePointer_mod, only : t_GmiArrayBundle !use GmiSpcConcentrationMethod_mod, only : isFixedConcentration ! diff --git a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt index 11904bb..17551ee 100644 --- a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt +++ b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CMakeLists.txt @@ -12,6 +12,6 @@ endif () esma_add_library (${this} SRCS ${SRCS} - DEPENDENCIES FVdycoreCubed_GridComp GMIchem_GridComp Chem_Shared MAPL_Base GMAO_hermes + DEPENDENCIES FVdycoreCubed_GridComp GMIchem_GridComp Chem_Shared MAPL GMAO_hermes ) diff --git a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 index 09afb1b..8fc25f1 100644 --- a/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/CTMpTracers_GridComp/CTM_pTracersGridCompMod.F90 @@ -13,8 +13,7 @@ module CTM_pTracersGridCompMod ! !USES: use ESMF use m_set_eta, only: set_eta - use MAPL_Mod - use MAPL_GenericMod + use MAPL USE Chem_UtilMod USE jw, only : tracer_q, tracer_q1_q2, tracer_q3 diff --git a/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 b/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 index 4a91d08..d05c203 100644 --- a/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 +++ b/src/Components/GEOSctm_GridComp/GEOS_ctmEnvGridComp.F90 @@ -12,7 +12,7 @@ module GEOS_ctmEnvGridComp ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use FV_StateMod, only : calcCourantNumberMassFlux => fv_computeMassFluxes use fv_arrays_mod , only: FVPRC use m_set_eta, only : set_eta diff --git a/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 b/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 index d3ca6d9..0cf72b7 100644 --- a/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 +++ b/src/Components/GEOSctm_GridComp/GEOS_ctmGridCompMod.F90 @@ -14,7 +14,7 @@ module GEOS_ctmGridCompMod ! ! !USES: use ESMF - use MAPL_Mod + use MAPL use Bundle_IncrementMod use GEOS_ctmEnvGridComp, only : EctmSetServices => SetServices diff --git a/src/Shared/.gitignore b/src/Shared/.gitignore index 9da6443..023c268 100644 --- a/src/Shared/.gitignore +++ b/src/Shared/.gitignore @@ -1,3 +1,9 @@ /@GMAO_Shared +/GMAO_Shared +/GMAO_Shared@ /@MAPL +/MAPL +/MAPL@ /@FMS +/FMS +/FMS@