From c65ca5c7266e25b16efbe7b3fb4e77026937a11e Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 7 Nov 2024 17:30:06 -0500 Subject: [PATCH] trigger o-server when running M21C; add replay input file to replay from M21C - an example --- src/Applications/GEOSdas_App/fvsetup | 22 +- .../GEOSdas_App/testsuites/m21c_j98.input | 4 +- .../GEOSdas_App/testsuites/m21c_j98RPY.input | 234 ++++++++++++++++++ .../GEOSdas_App/testsuites/x0048.input | 4 +- .../GEOSdas_App/testsuites/x0048RPY.input | 4 +- 5 files changed, 254 insertions(+), 14 deletions(-) create mode 100644 src/Applications/GEOSdas_App/testsuites/m21c_j98RPY.input diff --git a/src/Applications/GEOSdas_App/fvsetup b/src/Applications/GEOSdas_App/fvsetup index 2365682d..e22da3cb 100755 --- a/src/Applications/GEOSdas_App/fvsetup +++ b/src/Applications/GEOSdas_App/fvsetup @@ -351,6 +351,7 @@ use File::Path qw(mkpath rmtree); use Getopt::Long qw(GetOptions); use Sys::Hostname; use Time::Local; # time functions +use POSIX qw(ceil); use FindBin; # so we can find where this script resides use lib ("$FindBin::Bin", "$ESMADIR/$ARCH/bin"); @@ -3232,8 +3233,8 @@ sub get_times { # # ---- if ($res eq "C48") { $nx = 4; $ny = 24 } # (b) 96 elsif ($res eq "C90") { $nx = 4; $ny = 24 } # (c) 96 - elsif ($res eq "C180") { $nx = 15; $ny = 36 } # (d) 360 - elsif ($res eq "C360") { $nx = 8; $ny = 48 } # (e) 384 + elsif ($res eq "C180") { $nx = 9; $ny = 30 } # (d) 270 + elsif ($res eq "C360") { $nx = 10; $ny = 36 } # (e) 360 else { $nx = 28; $ny = 48 } # (f+) 1344 } else { @@ -4872,7 +4873,6 @@ EOF $use_shmem = 0; $o_servers = 0; $bckend_wrts = 0; - $ios_nds = 1; $cldmicro = "1MOMENT"; if ( "$res" eq "c" && "$vres" eq "55" ) { $anahgrd = substr($res,0,1); @@ -5167,7 +5167,6 @@ EOF $bckend_wrts = 24; } $cubed = 1; -# $ios_nds = 2; $specres = "254"; $jcap = "254"; $agcm_grid_type = "Cubed-Sphere"; @@ -5201,8 +5200,11 @@ EOF $ana_im_ens = 288; $ana_jm_ens = 181; } elsif ( "$res" eq "C360" ) { # Cubed-sphere + if ($r21c) { + $o_servers = 3; + $bckend_wrts = 24; + } $cubed = 1; -# $ios_nds = 2; $specres = "254"; $jcap = "254"; $agcm_grid_type = "Cubed-Sphere"; @@ -5276,7 +5278,6 @@ EOF } elsif ( "$res" eq "C1440" ) { # Cubed-sphere $cubed = 1; $o_servers = 8; -# $ios_nds = 4; $specres = "254"; $jcap = "254"; $agcm_grid_type = "Cubed-Sphere"; @@ -7789,7 +7790,8 @@ if ( $fvchem && $lm > 2 ) { } $pi = $i; # p for PSAS - my $xncpus = $ncpus + $o_servers * $ncpus_per_node; + my $this_many_nodes = ceil($ncpus/$ncpus_per_node) + $o_servers; + my $xncpus = $this_many_nodes * $ncpus_per_node; # Turn off some parallelization on discover #------------------------------------------ @@ -9887,7 +9889,11 @@ sub init_agcm_rc { AGCM_label_subst("\@GRID_TYPE", $agcm_grid_type); AGCM_label_subst("\@CUBE_AGCM", $cube_agcm); AGCM_label_subst("\@LATLON_AGCM", $latlon_agcm); - AGCM_label_subst("\@NUM_OSERVER_NODES", $ios_nds); + if ( $o_servers > 0 ) { + AGCM_label_subst("\@NUM_OSERVER_NODES", $o_servers); + } else { + AGCM_label_subst("\@NUM_OSERVER_NODES", 1); + } AGCM_label_subst("\@NUM_BACKEND_PES", $bckend_wrts); AGCM_label_subst("\@AANA_IM", $ana_im); diff --git a/src/Applications/GEOSdas_App/testsuites/m21c_j98.input b/src/Applications/GEOSdas_App/testsuites/m21c_j98.input index 9c692e06..31c2fb3b 100644 --- a/src/Applications/GEOSdas_App/testsuites/m21c_j98.input +++ b/src/Applications/GEOSdas_App/testsuites/m21c_j98.input @@ -98,10 +98,10 @@ Length of FORECAST run segments (in hours)? [123] Number of one-day DAS segments per PBS job? [1] > -Number of PEs in the zonal direction (NX)? [8] +Number of PEs in the zonal direction (NX)? [10] > -Number of PEs in the meridional direction (NY)? [48] +Number of PEs in the meridional direction (NY)? [36] > Job nickname? [g5das] diff --git a/src/Applications/GEOSdas_App/testsuites/m21c_j98RPY.input b/src/Applications/GEOSdas_App/testsuites/m21c_j98RPY.input new file mode 100644 index 00000000..a86364df --- /dev/null +++ b/src/Applications/GEOSdas_App/testsuites/m21c_j98RPY.input @@ -0,0 +1,234 @@ +#--------------- +# m21c_j98RPY.input +#--------------- + +codeID: 271edc7 +description: m21c_j98RPY__agrid_C360__ogrid_CS +fvsetupID: + +---ENDHEADERS--- + +Remote account for Intranet plots? [rtodling@train] +> + +Is this a MERRA2 (1), GEOSIT (2), R21C (3) experiment? [0] +> 3 + +AGCM Horizontal Resolution? [C360] +> C360 + +AGCM Vertical Resolution? [72] +> + +OGCM Resolution? [C] +> CS + +EXPID? [u000_C360] +> $expid + +Check for previous use of expid (y/n)? [y] +> n + +EXPDSC? [m21c_j98RPY__agrid_C360__ogrid_CS] +> + +Land Boundary Conditions? [Icarus-NLv3] +> + +Catchment Model choice? [1] +> + +FVHOME? [/discover/nobackup/rtodling/m21c_j98RPY] +> /discover/nobackup/projects/gmao/dadev/rtodling/M21C/$expid + +The directory /discover/nobackup/projects/gmao/dadev/rtodling/M21C/m21c_j98RPY does not exist. Create it now? [y] +> + +Processing nodes (1:Haswell, 2:Skylake, 3:Cascase, 4:Milan)? [3] +> 4 + +Which case of variational analysis? [1] +> + +Window of the variational analysis (min)? [360] +> + +FVINPUT? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar] +> + +REAL TIME BCS? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5gcm/bcs/realtime/OSTIA_REYNOLDS] +> + +agcmpert? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/agcmpert] +> + +chemistry? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/chemistry] +> + +g5chem? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5chem] +> + +g5gcm? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5gcm] +> + +PIESA? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/PIESA] +> + +MERRA2? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/MERRA2] +> + +AeroCom? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/AeroCom] +> + +FVICS? [/archive/u/jstassi/restarts/GEOSadas-5_24_0] +> /home/dao_ops/e5303_m21c_jan98/run/.../archive/rs/Y1997/M12/e5303_m21c_jan98.rst.19971230_21z.tar + +Run model-adjoint-related applications (0=no,1=yes)? [0] +> 0 + +Run analysis-sensitivity applications (0=no,1=yes)? [0] +> + +Ending year-month-day? [20190119] +> 19980206 + +Length of FORECAST run segments (in hours)? [123] +> + +Number of one-day DAS segments per PBS job? [1] +> + +Number of PEs in the zonal direction (NX)? [10] +> + +Number of PEs in the meridional direction (NY)? [36] +> + +Job nickname? [g5das] +> m21cA + +Run in split executable mode (1=yes;0=no)? [1] +> + +Frequency of background fields (min)? [60] +> + +Triangular spectral truncation? [254] +> + +Analysis vertical levels (sig))? [72] +> + +GSI grid resolution? [NA] +> + +GEOS grid resolution instead? [d] +> + +Ensemble Hybrid (<0;3=3dHyb;4=Hyb4d)? [4] +> + +Number of procs in the zonal direction (NX)? [12] +> + +Number of procs in the meridional direction (NY)? [20] +> + +Which main class of ObsSys (1: NRT; 2: MERRA; 3: MERRA-2; 4: GEOS-IT; 5: R21C)? [5] +> + +OBSERVING SYSTEM CLASSES? +> r21c_igra2_raob_pre-qc_bufr,r21c_cdas_conv_pre-qc_bufr,r21c_prof_pre-qc_bufr,r21c_avhrrwnd_pre-qc_bufr,r21c_nmodis_pre-qc_bufr,r21c_goes_pre-qc_prep_bufr,r21c_metsat_pre-qc_prep_bufr,r21c_jma_pre-qc_prep_bufr,r21c_ascat_pre-qc_bufr,r21c_repro_ers2_pre-qc_bufr,r21c_qscat_jpl_pre-qc_bufr,r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr + +CHECKING OBSYSTEM? [2] +> 1 + +Which RADCOR option? [HAIMB_HRAD] +> + +Use sat channel-correlated observation errors (y/n)? [y] +> + +Use aircraft bias correction (y/n)? [y] +> + +Use unified radiance bias correction (y/n)? [y] +> + +Land DAS Analysis (y/n)? [n] +> + +Frequency (in days) for writing restarts? [0] +> + +Frequency for PROGNOSTIC fields? [010000] +> + +Frequency for surface (2D) DIAGNOSTIC fields? [010000] +> + +Frequency for upper air (3D) DIAGNOSTIC fields? [030000] +> + +Dimension of output in zonal direction? [1152] +> + +Dimension of output in meridional direction? [721] +> + +Would you like 2D diagnostics? [y] +> + +Would you like 3D diagnostics? [y] +> + +Would you like to compress diagnostics output files? [n] +> + +Include GOCART tracers (CO,CO2,aerosols,etc) (y/n)? [y] +> + +Select GOCART Emission Files to use: [AMIP] +> + +Do Aerosol Analysis (y/n)? [y] +> + +AOD OBSERVING CLASSES [or type 'none']? +> patmosx_ods,mod04_061_his,myd04_061_his,aeronet_obs + +Enable GAAS feedback to model (y/n)? [y] +> + +Which template? [HISTORY_R21C.rc.tmpl] +> + +Which template? [GCMPROG.rc.tmpl] +> + +Output Restart TYPE (bin or nc4) [nc4] +> + +Select group: [g2538] +> g0613 + +Replayed Ensemble (from OPS)? [no] +> yes + +Replay exp name? [x0046d] +> e5303_m21c_jan98 + +Replay archive directory? [/discover/nobackup/projects/gmao/dadev/rtodling/archive/x0046d] +> /gpfsm/dnb08/projects/p253/archive/e5303_m21c_jan98 + +Edit COLLECTIONS list in run/HISTORY.rc.tmpl (y/n)? [n] +> + +Edit COLLECTIONS list in fcst/HISTORY.rc.tmpl (y/n)? [n] +> + +Edit files in run directory for CERES configuration (y/n)? [n] +> + +Which? [Q] +> diff --git a/src/Applications/GEOSdas_App/testsuites/x0048.input b/src/Applications/GEOSdas_App/testsuites/x0048.input index 419ae60a..b2f51b95 100644 --- a/src/Applications/GEOSdas_App/testsuites/x0048.input +++ b/src/Applications/GEOSdas_App/testsuites/x0048.input @@ -44,8 +44,8 @@ FVHOME? [/discover/nobackup/dao_it/x0048] The directory /discover/nobackup/projects/gmao/obsdev/dao_it/x0048 already exists. Clean it? [y] > -Processing nodes (1:Westmere, 2:SandyBridge, 3:Ivy Bridge, 4:Haswell, 5:Skylake, 6:Cascase)? [4] -> 5 +Processing nodes (1:Haswell, 2:Skylake, 3:Cascase, 4:Milan)? [4] +> 2 Which case of variational analysis? [1] > diff --git a/src/Applications/GEOSdas_App/testsuites/x0048RPY.input b/src/Applications/GEOSdas_App/testsuites/x0048RPY.input index 0e98c59e..bd614aae 100644 --- a/src/Applications/GEOSdas_App/testsuites/x0048RPY.input +++ b/src/Applications/GEOSdas_App/testsuites/x0048RPY.input @@ -44,8 +44,8 @@ FVHOME? [/discover/nobackup/dao_it/x0048RPY] The directory /discover/nobackup/projects/gmao/obsdev/dao_it/x0048RPY already exists. Clean it? [y] > -Processing nodes (1:Westmere, 2:SandyBridge, 3:Ivy Bridge, 4:Haswell, 5:Skylake, 6:Cascase)? [4] -> +Processing nodes (1:Haswell, 2:Skylake, 3:Cascase, 4:Milan)? [4] +> 2 Which case of variational analysis? [1] >