diff --git a/src/Applications/LDAS_App/CMakeLists.txt b/src/Applications/LDAS_App/CMakeLists.txt index affdbf63..9eb03f86 100644 --- a/src/Applications/LDAS_App/CMakeLists.txt +++ b/src/Applications/LDAS_App/CMakeLists.txt @@ -22,7 +22,7 @@ ecbuild_add_executable ( set (scripts ldas_setup process_hist.csh - process_rst.csh + process_rst.py ens_forcing/average_ensemble_forcing.py ens_forcing/ensemble_forc.py ens_forcing/regrid_forc.csh diff --git a/src/Applications/LDAS_App/ldas_setup b/src/Applications/LDAS_App/ldas_setup index 68a3480d..0854efa9 100755 --- a/src/Applications/LDAS_App/ldas_setup +++ b/src/Applications/LDAS_App/ldas_setup @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import sys @@ -16,7 +16,9 @@ from datetime import datetime from datetime import timedelta from collections import OrderedDict from dateutil.relativedelta import relativedelta - +from remap_utils import * +from remap_catchANDcn import * +from process_rst import * """ This script is intended to be run from any installed directory with GEOSldas.x and ldas_setup @@ -105,11 +107,12 @@ class LDASsetup: self.assim = False self.has_landassim_seed = False self.has_geos_pert = False - self.has_ldassa_pert = False self.nSegments = 1 self.perturb = 0 self.first_ens_id = 0 self.ladas_coupling = 0 + self.in_rstfile = None + self.in_tilefile = 'None' # default string # ------ # Read exe input file which is required to set up the dir # ------ @@ -138,7 +141,7 @@ class LDASsetup: # print rqd exe inputs if self.verbose: - print '\nInputs from execfile:\n' + print ('\nInputs from execfile:\n') _printdict(self.rqdExeInp) # nens is an integer and =1 for model run @@ -183,7 +186,7 @@ class LDASsetup: ) if self.rqdExeInp['RESTART'].isdigit() : if int(self.rqdExeInp['RESTART']) == 0 : - print "No restart file (cold restart): Forcing start date to January 1, 0z" + print ("No restart file (cold restart): Forcing start date to January 1, 0z") year = self.begDates[0].year self.begDates[0]=datetime(year =year,month=1,day =1,hour =0, minute= 0,second= 0) @@ -223,14 +226,14 @@ class LDASsetup: _difftime =timedelta(days = _years*365+_months*30+_days,hours = _hours,minutes=_mins,seconds=_seconds) _difftime = int(self.rqdExeInp['NUM_SGMT'])*_difftime - print int(self.rqdExeInp['NUM_SGMT']) + print (int(self.rqdExeInp['NUM_SGMT'])) _d = self.begDates[0] _endDate = self.endDates[0] _d = _d + _difftime while _d < _endDate : - print _difftime.days + print (_difftime.days) self.nSegments +=1 - print _d.year, _d.month, _d.day + print (_d.year, _d.month, _d.day) self.begDates.append(_d) self.endDates.insert(-1,_d) _d = _d+ _difftime @@ -292,6 +295,15 @@ class LDASsetup: ldas_domain = self.rqdExeInp['RESTART_PATH']+ \ self.rqdExeInp['RESTART_ID'] + \ '/output/'+self.rqdExeInp['RESTART_DOMAIN']+'/rc_out/'+self.rqdExeInp['RESTART_ID']+'.ldas_domain.txt' + inpdir=self.rqdExeInp['RESTART_PATH']+self.rqdExeInp['RESTART_ID']+'/input/' + in_tilefiles_ = glob.glob(inpdir+'*tile.data') + if len(in_tilefiles_) == 0 : + inpdir=self.rqdExeInp['RESTART_PATH']+self.rqdExeInp['RESTART_ID']+'/output/'+self.rqdExeInp['RESTART_DOMAIN']+'/rc_out/' + in_tilefiles_ = glob.glob(inpdir+'MAPL_*.til') + if len(in_tilefiles_) == 0 : + in_tilefiles_ = glob.glob(inpdir+'/*.til') + self.in_tilefile =os.path.realpath(in_tilefiles_[0]) + if os.path.isfile(ldas_domain) : _numd = int(linecache.getline(ldas_domain, 1).strip()) self.rqdExeInp['TILING_FILE'] =glob.glob(self.rqdExeInp['BCS_PATH']+'*.til')[0] @@ -338,12 +350,12 @@ class LDASsetup: _domain_dic['EXCLUDE_FILE']= "''" _domain_dic['INCLUDE_FILE']= "''" - for key,val in _domain_dic.iteritems() : + for key,val in _domain_dic.items() : if key in self.rqdExeInp : _domain_dic[key]= self.rqdExeInp[key] fout =open('LDAS_domain_def.nml','w') fout.write('&domain_inputs\n') - for key,val in _domain_dic.iteritems() : + for key,val in _domain_dic.items() : keyn=(key+" = ").ljust(16) valn = str(val) if '_FILE' in key: @@ -362,13 +374,9 @@ class LDASsetup: tmpRstDir=self.rqdExeInp['RESTART_PATH']+'/'.join([self.rqdExeInp['RESTART_ID'],'output', self.rqdExeInp['RESTART_DOMAIN'],'rs',self.ensdirs[0],y4m2]) catchRstFile=tmpRstDir+'/'+tmpFile - ldassa_tmp=self.rqdExeInp['RESTART_ID']+'.ens0000.'+self.catch+'_ldas_rst.'+y4m2d2_h2m2+'z.bin' - ldassaCN_tmp=self.rqdExeInp['RESTART_ID']+'.ens0000.'+self.catch+'_ldas_rst.'+y4m2d2_h2m2+'z' - LDASsa_catchRstFile=tmpRstDir+'/'+ldassa_tmp - LDASsa_CNRstFile=tmpRstDir+'/'+ldassaCN_tmp - assert os.path.isfile(catchRstFile) or os.path.isfile(LDASsa_catchRstFile) or os.path.isfile(LDASsa_CNRstFile), \ - self.catch+'_internal_rst file [%s] or [%s] does not exist!' %(catchRstFile, LDASsa_catchRstFile) + assert os.path.isfile(catchRstFile), self.catch+'_internal_rst file [%s] does not exist!' %(catchRstFile) + self.in_rstfile = catchRstFile if int(self.rqdExeInp['RESTART']) == 1 : tmpFile=self.rqdExeInp['RESTART_ID']+'.vegdyn_internal_rst' @@ -384,12 +392,26 @@ class LDASsetup: landpertRstFile=tmpRstDir+'/'+tmpFile if ( os.path.isfile(landpertRstFile)) : self.has_geos_pert = True - else : - ldassa_tmp=self.rqdExeInp['RESTART_ID']+'.ens0000.pert_ldas_rst.'+y4m2d2_h2m2+'z.bin' - LDASsa_pertRstFile=tmpRstDir+'/'+ldassa_tmp - if (os.path.isfile(LDASsa_pertRstFile)) : - self.has_ldassa_pert = True - + + elif (int(self.rqdExeInp['RESTART']) == 0) : + if (self.catch == 'catch'): + self.in_rstfile = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \ + '/Catch/M09/20170101/catch_internal_rst' + self.in_tilefile = '/discover/nobackup/projects/gmao/ssd/land/l_data/geos5/bcs/CLSM_params' \ + '/mkCatchParam_SMAP_L4SM_v002/SMAP_EASEv2_M09/SMAP_EASEv2_M09_3856x1624.til' + elif (self.catch == 'catchcnclm40'): + self.in_rstfile = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \ + '/CatchCN/M36/20150301_0000/catchcnclm40_internal_dummy' + self.in_tilefile = '/discover/nobackup/ltakacs/bcs/Heracles-NL/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til' + elif (self.catch == 'catchcnclm45'): + self.in_rstfile = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \ + '/CatchCN/M36/19800101_0000/catchcnclm45_internal_dummy' + self.in_tilefile = '/discover/nobackup/ltakacs/bcs/Icarus-NLv3/Icarus-NLv3_EASE/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til' + else: + sys.exit('need to provide at least dummy files') + self.in_rstfile = None + self.in_tilefile = None + # DEAL WITH mwRTM input from exec self.assim = True if self.rqdExeInp.get('LAND_ASSIM', 'NO').upper() == 'YES' else False # verify mwrtm file @@ -437,7 +459,7 @@ class LDASsetup: # print rqd rm inputs if self.verbose: - print '\n\nRequired inputs for resource manager:' + print ('\n\nRequired inputs for resource manager:') _printdict(self.rqdRmInp) # OPTIONAL inputs @@ -448,7 +470,7 @@ class LDASsetup: # print opt rm inputs if self.verbose: - print '\n\nOptional inputs for resource manager:' + print ('\n\nOptional inputs for resource manager:') _printdict(self.optRmInp) # ------ @@ -483,7 +505,7 @@ class LDASsetup: my_ntasks_per_node = int(self.rqdRmInp['ntasks-per-node']) # default number of nodes - my_nodes = self.rqdRmInp['ntasks_model'] / my_ntasks_per_node + my_nodes = self.rqdRmInp['ntasks_model'] // my_ntasks_per_node if self.rqdRmInp['ntasks_model'] % my_ntasks_per_node > 0 : my_nodes = my_nodes + 1 @@ -539,7 +561,7 @@ class LDASsetup: key = key.strip() val = val.strip() if not key or not val: - print "WARNING: " + errstr % (linenum, inpfile) + print ("WARNING: " + errstr % (linenum, inpfile)) continue #raise Exception(errstr % (linenum, inpfile)) if key in inpdict: @@ -663,10 +685,10 @@ class LDASsetup: tile=newtile # if three extra lines exist, remove them and save it to inputdir - print '\nCorrect the tile file if it is an old EASE tile format... \n' + print ('\nCorrect the tile file if it is an old EASE tile format... \n') EASEtile=self.bcsdir+'/MAPL_'+short_tile cmd = './preprocess_ldas.x correctease '+ tile + ' '+ EASEtile - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) if os.path.isfile(EASEtile) : @@ -699,8 +721,8 @@ class LDASsetup: cmd = './preprocess_ldas.x c_f2g ' + tile + ' ' + domain_def + ' '+ self.out_path + ' ' + catchment_def + ' ' + exp_id + ' ' + _y4m2d2h2m2 + ' '+ dzsf - print 'Creating f2g.txt....\n' - print "cmd: " + cmd + print ('Creating f2g.txt....\n') + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) # check if it is local or global with open('f2g.txt') as f2gfile : @@ -711,10 +733,10 @@ class LDASsetup: # update tile domain if self.islocal: newlocalTile = tile+'.domain' - print "\nCreating local tile file :"+ newlocalTile - print "\n by excluding land type MAPL_Land_ExcludeFromDomain=1100...\n" + print ("\nCreating local tile file :"+ newlocalTile) + print ("\n by excluding land type MAPL_Land_ExcludeFromDomain=1100...\n") cmd = './preprocess_ldas.x c_localtile ' + tile + ' ' + newlocalTile - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) short_tile=short_tile +'.domain' tile = newlocalTile @@ -739,18 +761,18 @@ class LDASsetup: bcs=bcstmp if self.islocal: - print "Creating the boundary files for the simulation domain...\n" + print ("Creating the boundary files for the simulation domain...\n") bcs_tmp=[] for bcf in bcs : cmd = './preprocess_ldas.x c_localbc ' + bcf + ' '+ bcf+'.domain' - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) bcs_tmp=bcs_tmp+[bcf+'.domain'] bcs=bcs_tmp # link BC - print "linking bcs..." + print ("linking bcs...") bcnames=['green','lai','ndvi','nirdf','visdf'] if (self.rqdExeInp['LNFM_FILE'] != ''): bcnames += ['lnfm'] @@ -761,7 +783,7 @@ class LDASsetup: os.symlink(bc,myBC) # create and link restart - print "Creating and lining restart..." + print ("Creating and linking restart...") _start = self.begDates[0] y4m2='Y%4d/M%02d'%(_start.year, _start.month) @@ -780,9 +802,9 @@ class LDASsetup: sponsorid = self.rqdRmInp['account'] exp_id = self.rqdExeInp['EXP_ID'] exp_dir = self.exphome - bcdir = self.rqdExeInp['BCS_PATH'] - tilefile = os.path.basename(self.rqdExeInp['TILING_FILE']) - have_rst = str(self.rqdExeInp['RESTART']) + out_bcdir = self.rqdExeInp['BCS_PATH'] + out_tilefile = self.rqdExeInp['TILING_FILE'] + RESTART_str = str(self.rqdExeInp['RESTART']) YYYYMMDD = '%4d%02d%02d' % (_start.year, _start.month,_start.day) YYYYMMDDHH= '%4d%02d%02d%02d' % (_start.year, _start.month,_start.day, _start.hour) rstid = self.rqdExeInp['RESTART_ID'] @@ -799,23 +821,51 @@ class LDASsetup: shutil.copy(landassim_seeds, _seeds) os.symlink(_seeds, myRstDir+ '/landassim_obspertrseed'+ _ensid +'_rst') self.has_landassim_seed = True - - cmd= ' '.join(['./process_rst.csh', sponsorid, exp_id, exp_dir, - bcdir, tilefile, self.catch, have_rst, YYYYMMDDHH, - rstid, rstdomain, rstpath0, str(self.nens), str(self.rqdExeInp['RUN_IRRIG']), - dzsf, wemin_in, wemin_out]) - print "cmd: " + cmd - os.system(cmd) - - done_rst=self.exphome+'/'+exp_id+'/mk_restarts/done_rst_file' - print "Please hold on for a while until the restart file is created ....." - _animation = "|/-\\" - _idx = 0 - while not os.path.isfile(done_rst): - sys.stdout.write('\r'+_animation[_idx % len(_animation)]) - sys.stdout.flush() - _idx += 1 - time.sleep(1.) + mk_outdir = self.exphome+'/'+exp_id+'/mk_restarts/' + cmd= ' '.join(['./process_rst.csh', sponsorid, exp_id, mk_outdir, + out_bcdir, out_tilefile, self.catch, RESTART_str, YYYYMMDDHH, + self.in_rstfile, self.in_tilefile, dzsf, wemin_in, wemin_out]) + + if (RESTART_str != '1'): + remap_tpl = os.path.dirname(os.path.realpath(__file__)) + '/remap_params.tpl' + config = yaml_to_config(remap_tpl) + + config['slurm']['account'] = self.rqdRmInp['account'] + config['slurm']['qos'] = 'debug' + config['slurm']['qos'] = 'cas' + + config['input']['surface']['catch_tilefile'] = self.in_tilefile + config['input']['shared']['expid'] = self.rqdExeInp['RESTART_ID'] + config['input']['shared']['yyyymmddhh'] = YYYYMMDDHH + config['input']['shared']['rst_dir'] = os.path.dirname(self.in_rstfile)+'/' + config['input']['surface']['wemin'] = wemin_in + config['input']['surface']['catch_model'] = self.catch + + config['output']['shared']['out_dir'] = mk_outdir + config['output']['surface']['catch_remap'] = True + config['output']['surface']['catch_tilefile'] = self.rqdExeInp['TILING_FILE'] + config['output']['shared']['bcs_dir'] = self.rqdExeInp['BCS_PATH'] + config['output']['shared']['expid'] = self.rqdExeInp['EXP_ID'] + config['output']['surface']['surflay'] = dzsf + config['output']['surface']['wemin'] = wemin_out + + config = remap_config_ldas( config, RESTART_str, self.rqdExeInp['RESTART_PATH'], self.rqdExeInp['RESTART_ID']) + + catch_obj = catchANDcn(config_obj = config) + catch_obj.remap() + + #print "cmd: " + cmd + #os.system(cmd) + + #done_rst=self.exphome+'/'+exp_id+'/mk_restarts/done_rst_file' + #print "Please hold on for a while until the restart file is created ....." + #_animation = "|/-\\" + #_idx = 0 + #while not os.path.isfile(done_rst): + # sys.stdout.write('\r'+_animation[_idx % len(_animation)]) + # sys.stdout.flush() + # _idx += 1 + # time.sleep(1.) #for ens in self.ensdirs : @@ -831,36 +881,30 @@ class LDASsetup: catchRstFile = '' vegdynRstFile = '' pertRstFile = '' - print "restart: " + self.rqdExeInp['RESTART'] + print ("restart: " + self.rqdExeInp['RESTART']) + if self.rqdExeInp['RESTART'].isdigit() : + if int(self.rqdExeInp['RESTART']) == 0 or int(self.rqdExeInp['RESTART']) == 2 : vegdynRstFile = glob.glob(self.rqdExeInp['BCS_PATH']+'vegdyn_*.dat')[0] - catchRstFile = self.exphome+'/'+exp_id+'/mk_restarts/'+self.catch+'_internal_rst.'+YYYYMMDD - else : + catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0] + else : # RESTART == 1 catchRstFile = rstpath+ens +'/'+ y4m2+'/'+self.rqdExeInp['RESTART_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2 - _catchRstFile = self.exphome+'/'+exp_id+'/mk_restarts/'+self.catch+ensid+'_internal_rst.'+YYYYMMDD vegdynRstFile= rstpath+ens +'/'+self.rqdExeInp['RESTART_ID']+ '.vegdyn_internal_rst' if not os.path.isfile(vegdynRstFile): # no vegdyn restart from LDASsa - if not os.path.isfile(vegdynRstFile0): - vegdynRstFile = glob.glob(self.rqdExeInp['BCS_PATH']+'vegdyn_*.dat')[0] - if (self.nens == 1) : - _catchRstFile = self.exphome+'/'+exp_id+'/mk_restarts/'+self.catch+'0000_internal_rst.'+YYYYMMDD - if os.path.isfile(_catchRstFile): # from LDASsa restart - catchRstFile = _catchRstFile - assert int(self.rqdExeInp['RST_FROM_GLOBAL']) == 1, 'restart from LDASsa should be global' + if not os.path.isfile(vegdynRstFile0): + vegdynRstFile = glob.glob(self.rqdExeInp['BCS_PATH']+'vegdyn_*.dat')[0] else : vegdynRstFile = glob.glob(self.rqdExeInp['BCS_PATH']+'vegdyn_*.dat')[0] - catchRstFile = self.exphome+'/'+exp_id+'/mk_restarts/'+self.catch+'_internal_rst.'+YYYYMMDD + catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0] # catchment restart file - print 'catchRstFile: ' + catchRstFile if os.path.isfile(catchRstFile) : - catchLocal = self.rstdir+ens +'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2 if self.islocal : - print "Creating local catchment restart file... \n" + print( "Creating local catchment restart file... \n") cmd='./preprocess_ldas.x c_localcatchrst '+ catchRstFile +' ' + catchLocal - print "cmd: "+cmd + print ("cmd: "+cmd) sp.call(shlex.split(cmd)) else : shutil.copy(catchRstFile,catchLocal) @@ -869,17 +913,16 @@ class LDASsetup: if '0000' in ens : catchRstFile0 = catchRstFile - else : + else : # re-use 0000 catch file catchRstFile = catchRstFile0 # vegdyn restart file if os.path.isfile(vegdynRstFile) : - vegdynLocal = self.rstdir+ens +'/'+self.rqdExeInp['EXP_ID']+'.vegdyn_internal_rst' if self.islocal : - print "Creating the local veg restart file... \n" + print ("Creating the local veg restart file... \n") cmd='./preprocess_ldas.x c_localvegrst '+ vegdynRstFile +' ' + vegdynLocal - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) else : shutil.copy(vegdynRstFile,vegdynLocal) @@ -897,19 +940,11 @@ class LDASsetup: shutil.copy(pertRstFile,pertLocal) pertRstFile = pertLocal - if (self.has_ldassa_pert and self.perturb == 1 ) : - pertRstFile = rstpath+ens +'/'+ y4m2+'/'+self.rqdExeInp['RESTART_ID']+'.'+ens+'.pert_ldas_rst.'+y4m2d2_h2m2+'z.bin' - pertLocal = self.rstdir+ens +'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.landpert_internal_rst.'+y4m2d2_h2m2 - print "Convert LDASsa pert " + ensid + " rst to GEOSldas rst" - cmd = './preprocess_ldas.x c_convert_pert '+ pertRstFile + ' ' + pertLocal + ' ' + self.out_path + ' ' + self.rqdExeInp['EXP_ID'] - sp.call(shlex.split(cmd)) - pertRstFile = pertLocal - - print 'catchRstFile: ' + catchRstFile - + print ('catchRstFile: ' + catchRstFile) + print ('vegdynRstFile: ' + vegdynRstFile) os.symlink(catchRstFile, myCatchRst) os.symlink(vegdynRstFile, myVegRst) - if ( (self.has_geos_pert or self.has_ldassa_pert) and self.perturb == 1 ): + if ( self.has_geos_pert and self.perturb == 1 ): os.symlink(pertRstFile, myPertRst) # catch_param restar file @@ -920,9 +955,9 @@ class LDASsetup: mwRTMRstFile = self.mwrtm_file mwRTMLocal = self.bcsdir+'/'+ y4m2+'/'+self.rqdExeInp['EXP_ID']+'.ldas_mwRTMparam.'+y4m2d2_h2m2+'z.nc4' if self.islocal : - print "Creating the local mwRTM restart file... \n" + print ("Creating the local mwRTM restart file... \n") cmd='./preprocess_ldas.x c_localmwrtmrst '+ mwRTMRstFile +' ' + mwRTMLocal - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) else : shutil.copy(mwRTMRstFile,mwRTMLocal) @@ -932,7 +967,7 @@ class LDASsetup: os.symlink(mwRTMRstFile, mymwRTMRst) # update 'restart_path' to use relative path from outdir - print "Updating restart path..." + print ("Updating restart path...") self.rqdExeInp['RESTART_PATH'] = myRstDir if os.path.isfile('f2g.txt'): os.remove('f2g.txt') @@ -993,15 +1028,15 @@ class LDASsetup: if os.path.isfile('optimized_distribution'): os.remove('optimized_distribution') - print "Optimizing... decomposition of processes.... \n" + print ("Optimizing... decomposition of processes.... \n") cmd = './preprocess_ldas.x optimize '+ self.inpdir+'/tile.data '+ str(self.rqdRmInp['ntasks_model']) - print "cmd: " + cmd + print ("cmd: " + cmd) sp.call(shlex.split(cmd)) optinxny=self._parseInputFile('optimized_distribution') if (int(optinxny['NX']) == 1): if int(optinxny['NY']) != int(self.rqdRmInp['ntasks_model']): self.rqdRmInp['ntasks_model']=optinxny['NY'] - print 'adjust ntasks_model %d for cubed-sphere grid' % int(self.rqdRmInp['ntasks_model']) + print ('adjust ntasks_model %d for cubed-sphere grid' % int(self.rqdRmInp['ntasks_model'])) if os.path.isfile('IMS.rc') : shutil.move('IMS.rc', self.rundir+'/') @@ -1013,10 +1048,10 @@ class LDASsetup: # DEFAULT rc files default_rc = glob.glob(etcdir+'/GEOSldas_*.rc') assert len(default_rc)==4 - print default_rc + print (default_rc) for rcfile in default_rc: shortfile=rcfile.rsplit('GEOSldas_',1)[1] - print shortfile + ' ' + etcdir + ' ' + self.rundir + print (shortfile + ' ' + etcdir + ' ' + self.rundir) if shortfile =='HIST.rc': tmprcfile=self.rundir+'/HISTORY.rc' histrc_file=rcfile @@ -1044,7 +1079,7 @@ class LDASsetup: #os.system(cmd) sp.call(shlex.split(cmd)) for line in fileinput.input(tmprcfile,inplace=True): - print line.rstrip().replace('GEOSldas_expid',self.rqdExeInp['EXP_ID']) + print (line.rstrip().replace('GEOSldas_expid',self.rqdExeInp['EXP_ID'])) # just copy an empty ExtData.rc if shortfile=='ExtData.rc' : shutil.copy2(rcfile, self.rundir+'/'+shortfile) @@ -1056,33 +1091,33 @@ class LDASsetup: _num_sgmt = int(self.rqdExeInp['NUM_SGMT']) for line in fileinput.input(tmprcfile,inplace=True): - print line.rstrip().replace('JOB_SGMT:',self.job_sgmt[0]) + print (line.rstrip().replace('JOB_SGMT:',self.job_sgmt[0])) for line in fileinput.input(tmprcfile,inplace=True): - print line.rstrip().replace('NUM_SGMT:','NUM_SGMT: %d'% _num_sgmt) + print (line.rstrip().replace('NUM_SGMT:','NUM_SGMT: %d'% _num_sgmt)) for line in fileinput.input(tmprcfile,inplace=True): - print line.rstrip().replace('BEG_DATE:',self.begDates[0].strftime('BEG_DATE: %Y%m%d %H%M%S')) + print (line.rstrip().replace('BEG_DATE:',self.begDates[0].strftime('BEG_DATE: %Y%m%d %H%M%S'))) for line in fileinput.input(tmprcfile,inplace=True): - print line.rstrip().replace('END_DATE:',self.endDates[-1].strftime('END_DATE: %Y%m%d %H%M%S')) + print (line.rstrip().replace('END_DATE:',self.endDates[-1].strftime('END_DATE: %Y%m%d %H%M%S'))) if shortfile == 'LDAS.rc' : ldasrcInp = OrderedDict() # land default default_surfrcInp = self._parseInputFile(etcdir+'/GEOS_SurfaceGridComp.rc') - for key,val in default_surfrcInp.iteritems() : + for key,val in default_surfrcInp.items() : ldasrcInp[key] = val # ldas default, may overwrite land default default_ldasrcInp = self._parseInputFile(rcfile) - for key,val in default_ldasrcInp.iteritems() : + for key,val in default_ldasrcInp.items() : ldasrcInp[key] = val # exeinp, may overwrite ldas default - for key,val in self.rqdExeInp.iteritems(): + for key,val in self.rqdExeInp.items(): if key not in self.NoneLDASrcKeys: ldasrcInp[key]= val # overide by optimized distribution - #for key,val in optinxny.iteritems(): + #for key,val in optinxny.items(): # ldasrcInp[key]= val # create BC in rc file @@ -1163,10 +1198,10 @@ class LDASsetup: # write LDAS.rc fout =open(self.rundir+'/'+shortfile,'w') # ldasrcInp['NUM_LDAS_ENSEMBLE']=ldasrcInp.pop('NUM_ENSEMBLE') - for key,val in optinxny.iteritems(): + for key,val in optinxny.items(): keyn=(key+":").ljust(36) fout.write(keyn+str(val)+'\n') - for key,val in ldasrcInp.iteritems() : + for key,val in ldasrcInp.items() : keyn=(key+":").ljust(36) fout.write(keyn+str(val)+'\n') fout.write("OUT_PATH:".ljust(36)+self.out_path+'\n') @@ -1230,7 +1265,7 @@ class LDASsetup: fout.write("\nsed -i 's/#if($capdate<$enddate) sbatch/if($capdate<$enddate) sbatch /g' lenkf.j") fout.close() - os.chmod(self.rundir+'/ldas_batchrun.j', 0755) + sp.call(['chmod', '755', self.rundir+'/ldas_batchrun.j']) status = True return status @@ -1326,11 +1361,11 @@ class LDASsetup: else : fout.write(line.replace('MY_EXPDIR',self.exphome+'/$EXPID')) - os.chmod('lenkf.j', 0755) + sp.call(['chmod', '755', 'lenkf.j']) expdir = '/'.join(self.rundir.rstrip('/').split('/')[:-1]) - print '\nExperiment directory: %s' % expdir - print + print ('\nExperiment directory: %s' % expdir) + print () status = True return status @@ -1339,186 +1374,184 @@ def _printdict(d): Private method: print a 'flat' dictionary """ - for key, val in d.iteritems(): - print key.ljust(23), ':', val + for key, val in d.items(): + print (key.ljust(23), ':', val) def _printExeInputKeys(rqdExeInpKeys): """ Private method: print sample exe input """ - print '####################################################################################' - print '# #' - print '# REQUIRED INPUTS #' - print '# #' - print '# These inputs are needed to set up output dir structure. #' - print '# #' - print '####################################################################################' - print - print '############################################################' - print '# #' - print '# EXPERIMENT INFO #' - print '# #' - print '# Format for start/end times is yyyymmdd hhmmss. #' - print '# #' - print '############################################################' - print - print 'EXP_ID:' - print 'EXP_DOMAIN:' - print 'NUM_LDAS_ENSEMBLE:' - print 'BEG_DATE:' - print 'END_DATE:' - print - print '############################################################' - print '# #' - print '# RESTART INFO #' - print '# #' - print '# (i) Select "RESTART" option: #' - print '# #' - print '# Use one of the following options if you *have* a #' - print '# GEOSldas- or LDASsa-produced restart file: #' - print '# #' - print '# RESTART: 1 #' - print '# YES, have restart file from GEOSldas *or* LDASsa #' - print '# in SAME tile space (grid) with SAME boundary #' - print '# conditions and SAME snow model parameter (WEMIN). #' - print '# GEOSldas-produced restart can be for the same or #' - print '# a larger domain. #' - print '# LDASsa-produced restart *must* be for the GLOBAL #' - print '# domain. #' - print '# #' - print '# RESTART: 2 #' - print '# YES, have restart file from GEOSldas *or* LDASsa but #' - print '# in a DIFFERENT tile space (grid) or with #' - print '# DIFFERENT boundary conditions or DIFFERENT snow #' - print '# model parameter (WEMIN). #' - print '# Restart *must* be for the GLOBAL domain. #' - print '# #' - print '# Use one of the following options if you DO NOT have a #' - print '# GEOSldas- or LDASsa-produced restart file #' - print '# (works for global domain ONLY!): #' - print '# #' - print '# RESTART: 0 #' - print '# Cold start from some old restart for Jan 1, 0z. #' - print '# #' - print '# RESTART: M #' - print '# Re-tile from archived MERRA-2 restart file. #' - print '# #' - print '# RESTART: F #' - print '# Re-tile from FP (Forward Processing) restart file. #' - print '# #' - print '# RESTART: G #' - print '# Re-tile from any AGCM catch[cnclmxx]_internal_rst file.#' - print '# #' - print '# -------------------------------------------------------- #' - print '# IMPORTANT: #' - print '# Except for RESTART=1, SPIN-UP is REQUIRED in almost #' - print '# all cases. #' - print '# -------------------------------------------------------- #' - print '# #' - print '# #' - print '# (ii) Specify experiment ID/location of restart file: #' - print '# #' - print '# For RESTART=1 or RESTART=2: #' - print '# Specify RESTART_ID, RESTART_PATH, RESTART_DOMAIN with #' - print '# restarts stored as follows: #' - print '# RESTART_PATH/RESTART_ID/output/RESTART_DOMAIN/rs/ #' - print '# #' - print '# For RESTART=0 or RESTART=M or RESTART=F: #' - print '# There is no need to specify RESTART_ID, RESTART_PATH, #' - print '# and RESTART_DOMAIN. #' - print '# #' - print '# For RESTART=G: #' - print '# RESTART_ID : full_path_to_AGCM_experiment_directory #' - print '# RESTART_PATH : full_path_of_the_AGCM_restart_file #' - print '# RESTART_DOMAIN is NOT required. #' - print '# #' - print '############################################################' - print - print 'RESTART:' - print '#RESTART_ID:' - print '#RESTART_PATH:' - print '#RESTART_DOMAIN:' - print - print '############################################################' - print '# #' - print '# SURFACE METEOROLOGICAL FORCING #' - print '# #' - print '# Surface meteorological forcing time step is in seconds. #' - print '# #' - print '# For more information, see: #' - print '# GEOSldas/doc/README.MetForcing_and_BCS.md #' - print '# #' - print '############################################################' - print - print 'MET_TAG:' - print 'MET_PATH:' - print 'FORCE_DTSTEP:' - print - print '############################################################' - print '# #' - print '# LAND BOUNDARY CONDITIONS (BCS) #' - print '# #' - print '# Path to and (atmospheric) resolution of BCS. #' - print '# #' - print '# For more information, see: #' - print '# GEOSldas/doc/README.MetForcing_and_BCS.md #' - print '# [..]/GEOSsurface_GridComp/Utils/Raster/make_bcs #' - print '# #' - print '############################################################' - print - print 'BCS_PATH:' - print 'BCS_RESOLUTION:' - print - print '############################################################' - print '# #' - print '# LADAS COUPLING #' - print '# #' - print '# Coupling of LDAS to ADAS ("LADAS"): #' - print '# #' - print '# 0 -- LDAS not coupled with ADAS (default) #' - print '# 1 -- LDAS coupled with central member of ADAS #' - print '# 2 -- LDAS coupled with ens component of ADAS #' - print '# #' - print '# Requirements for LADAS_COUPLING > 0: #' - print '# #' - print '# (0) Specify ADAS_EXPDIR = [full_path]/[ADAS_EXPID] #' - print '# #' - print '# (1) BEG_DATE must be consistent with first cycle date #' - print '# and time of ADAS experiment (time is typically #' - print '# 3z, 9z, 15z, or 21z) #' - print '# #' - print '# (2) EXP_DOMAIN must be global CS grid as in ADAS exp #' - print '# #' - print '# (3) MET_TAG must be set to [ADAS_EXPID]__Nx+- #' - print '# MET_PATH must be set as follows for #' - print '# LADAS_COUPLING = 1: #' - print '# [full_path]/[LDAS_EXPID]/scratch/ #' - print '# LADAS_COUPLING = 2: #' - print '# [ADAS_EXPDIR]/atmens/ensdiag/forc #' - print '# After ldas exp setup, verify the following link: #' - print '# ../input/met_forcing/forc -> [MET_PATH] #' - print '# #' - print '# (4) BCS_PATH must be consistent with that of #' - print '# [ADAS_EXPDIR][/run/lnbcs #' - print '# #' - print '# (5) JOB_SGMT must match ADAS analysis window #' - print '# (typically 6h) #' - print '# #' - print '# (6) NUM_SGMT must be set to 1 #' - print '# #' - print '# (7) HISTORY: #' - print '# - instantaneous "catch_progn_incr" must be in #' - print '# HISTORY collection #' - print '# - time step must match that of LDAS analysis #' - print '# - for LADAS_COUPLING=2, HISTORY must include #' - print '# "catch_progn_incr[ENS_INDEX]" #' - print '# #' - print '############################################################' - print - print 'LADAS_COUPLING: 0' - print - print + print ('####################################################################################') + print ('# #') + print ('# REQUIRED INPUTS #') + print ('# #') + print ('# These inputs are needed to set up output dir structure. #') + print ('# #') + print ('####################################################################################') + print () + print ('############################################################') + print ('# #') + print ('# EXPERIMENT INFO #') + print ('# #') + print ('# Format for start/end times is yyyymmdd hhmmss. #') + print ('# #') + print ('############################################################') + print () + print ('EXP_ID:') + print ('EXP_DOMAIN:') + print ('NUM_LDAS_ENSEMBLE:') + print ('BEG_DATE:') + print ('END_DATE:') + print () + print ('############################################################') + print ('# #') + print ('# RESTART INFO #') + print ('# #') + print ('# (i) Select "RESTART" option: #') + print ('# #') + print ('# Use one of the following options if you *have* a #') + print ('# GEOSldas restart file: #') + print ('# #') + print ('# RESTART: 1 #') + print ('# YES, have restart file from GEOSldas #') + print ('# in SAME tile space (grid) with SAME boundary #') + print ('# conditions and SAME snow model parameter (WEMIN). #') + print ('# The restart domain can be for the same or #') + print ('# a larger one. #') + print ('# #') + print ('# RESTART: 2 #') + print ('# YES, have restart file from GEOSldas but #') + print ('# in a DIFFERENT tile space (grid) or with #') + print ('# DIFFERENT boundary conditions or DIFFERENT snow #') + print ('# model parameter (WEMIN). #') + print ('# Restart *must* be for the GLOBAL domain. #') + print ('# #') + print ('# Use one of the following options if you DO NOT have a #') + print ('# GEOSldas restart file #') + print ('# (works for global domain ONLY!): #') + print ('# #') + print ('# RESTART: 0 #') + print ('# Cold start from some old restart for Jan 1, 0z. #') + print ('# #') + print ('# RESTART: M #') + print ('# Re-tile from archived MERRA-2 restart file. #') + print ('# #') + print ('# RESTART: F #') + print ('# Re-tile from FP (Forward Processing) restart file. #') + print ('# #') + print ('# RESTART: G #') + print ('# Re-tile from any AGCM catch[cnclmxx]_internal_rst file.#') + print ('# #') + print ('# -------------------------------------------------------- #') + print ('# IMPORTANT: #') + print ('# Except for RESTART=1, SPIN-UP is REQUIRED in almost #') + print ('# all cases. #') + print ('# -------------------------------------------------------- #') + print ('# #') + print ('# #') + print ('# (ii) Specify experiment ID/location of restart file: #') + print ('# #') + print ('# For RESTART=1 or RESTART=2: #') + print ('# Specify RESTART_ID, RESTART_PATH, RESTART_DOMAIN with #') + print ('# restarts stored as follows: #') + print ('# RESTART_PATH/RESTART_ID/output/RESTART_DOMAIN/rs/ #') + print ('# #') + print ('# For RESTART=0 or RESTART=M or RESTART=F: #') + print ('# There is no need to specify RESTART_ID, RESTART_PATH, #') + print ('# and RESTART_DOMAIN. #') + print ('# #') + print ('# For RESTART=G: #') + print ('# RESTART_ID : full_path_to_AGCM_experiment_directory #') + print ('# RESTART_PATH : full_path_of_the_AGCM_restart_file #') + print ('# RESTART_DOMAIN is NOT required. #') + print ('# #') + print ('############################################################') + print () + print ('RESTART:') + print ('#RESTART_ID:') + print ('#RESTART_PATH:') + print ('#RESTART_DOMAIN:') + print () + print ('############################################################') + print ('# #') + print ('# SURFACE METEOROLOGICAL FORCING #') + print ('# #') + print ('# Surface meteorological forcing time step is in seconds. #') + print ('# #') + print ('# For more information, see: #') + print ('# GEOSldas/doc/README.MetForcing_and_BCS.md #') + print ('# #') + print ('############################################################') + print () + print ('MET_TAG:') + print ('MET_PATH:') + print ('FORCE_DTSTEP:') + print () + print ('############################################################') + print ('# #') + print ('# LAND BOUNDARY CONDITIONS (BCS) #') + print ('# #') + print ('# Path to and (atmospheric) resolution of BCS. #') + print ('# #') + print ('# For more information, see: #') + print ('# GEOSldas/doc/README.MetForcing_and_BCS.md #') + print ('# [..]/GEOSsurface_GridComp/Utils/Raster/make_bcs #') + print ('# #') + print ('############################################################') + print () + print ('BCS_PATH:') + print ('BCS_RESOLUTION:') + print () + print ('############################################################') + print ('# #') + print ('# LADAS COUPLING #') + print ('# #') + print ('# Coupling of LDAS to ADAS ("LADAS"): #') + print ('# #') + print ('# 0 -- LDAS not coupled with ADAS (default) #') + print ('# 1 -- LDAS coupled with central member of ADAS #') + print ('# 2 -- LDAS coupled with ens component of ADAS #') + print ('# #') + print ('# Requirements for LADAS_COUPLING > 0: #') + print ('# #') + print ('# (0) Specify ADAS_EXPDIR = [full_path]/[ADAS_EXPID] #') + print ('# #') + print ('# (1) BEG_DATE must be consistent with first cycle date #') + print ('# and time of ADAS experiment (time is typically #') + print ('# 3z, 9z, 15z, or 21z) #') + print ('# #') + print ('# (2) EXP_DOMAIN must be global CS grid as in ADAS exp #') + print ('# #') + print ('# (3) MET_TAG must be set to [ADAS_EXPID]__Nx+- #') + print ('# MET_PATH must be set as follows for #') + print ('# LADAS_COUPLING = 1: #') + print ('# [full_path]/[LDAS_EXPID]/scratch/ #') + print ('# LADAS_COUPLING = 2: #') + print ('# [ADAS_EXPDIR]/atmens/ensdiag/forc #') + print ('# After ldas exp setup, verify the following link: #') + print ('# ../input/met_forcing/forc -> [MET_PATH] #') + print ('# #') + print ('# (4) BCS_PATH must be consistent with that of #') + print ('# [ADAS_EXPDIR][/run/lnbcs #') + print ('# #') + print ('# (5) JOB_SGMT must match ADAS analysis window #') + print ('# (typically 6h) #') + print ('# #') + print ('# (6) NUM_SGMT must be set to 1 #') + print ('# #') + print ('# (7) HISTORY: #') + print ('# - instantaneous "catch_progn_incr" must be in #') + print ('# HISTORY collection #') + print ('# - time step must match that of LDAS analysis #') + print ('# - for LADAS_COUPLING=2, HISTORY must include #') + print ('# "catch_progn_incr[ENS_INDEX]" #') + print ('# #') + print ('############################################################') + print () + print ('LADAS_COUPLING: 0') + print () + print () _fn = '../etc/GEOSldas_LDAS.rc' # run ldas_setup from /bin directory @@ -1529,8 +1562,8 @@ def _printExeInputKeys(rqdExeInpKeys): sys.stdout.write(line) sys.stdout.flush() i_ += 1 - print - print + print () + print () _fn = '../etc/GEOS_SurfaceGridComp.rc' # run ldas_setup from /bin directory @@ -1549,42 +1582,42 @@ def _printExeInputKeys(rqdExeInpKeys): sys.stdout.write(line) sys.stdout.flush() i_ += 1 - print - print + print () + print () def _printRmInputKeys(rqdRmInpKeys, optRmInpKeys): """ Private method: print sample resource manager input """ - print '#' - print '# REQUIRED inputs' - print '#' - print '# NOTE:' - print '# - account = computational project number' - print '# [At NCCS: Use command "getsponsor" to see available account number(s).]' - print '# - walltime = walltime requested; format is HH:MM:SS (hours/minutes/seconds)' - print '# - ntasks_model = number of processors requested for the model (typically 112; output server is not included)' - print '# - ntasks-per-node = number of tasks per node (typically 46 for cascade lake*, 40 for skylake, and 28 for haswell nodes)' - print '# [If >40, cascade lake nodes will be allocated, if >28, cascade or skylake, else cascade, skylake or haswell.]' - print '# [*NCCS recommends <=46 cores per node on SCU16 (cascade lake) due to OS issues (as of 6 Oct 2021).]' - print '#' + print ('#') + print ('# REQUIRED inputs') + print ('#') + print ('# NOTE:') + print ('# - account = computational project number') + print ('# [At NCCS: Use command "getsponsor" to see available account number(s).]' ) + print ('# - walltime = walltime requested; format is HH:MM:SS (hours/minutes/seconds)') + print ('# - ntasks_model = number of processors requested for the model (typically 112; output server is not included)') + print ('# - ntasks-per-node = number of tasks per node (typically 46 for cascade lake*, 40 for skylake, and 28 for haswell nodes)') + print ('# [If >40, cascade lake nodes will be allocated, if >28, cascade or skylake, else cascade, skylake or haswell.]') + print ('# [*NCCS recommends <=46 cores per node on SCU16 (cascade lake) due to OS issues (as of 6 Oct 2021).]') + print ('#') for key in rqdRmInpKeys: - print key + ':' - print - print '#' - print '# OPTIONAL inputs' - print '#' - print '# NOTE:' - print '# - job_name = name of experiment; default is "exp_id"' - print '# - qos = quality-of-service; do not specify by default; specify "debug" for faster but limited service.' - print '# - oserver_nodes = number of nodes for oserver ( default is 0 )' - print '# - writers-per-node = tasks per oserver_node for writing ( default is 5 ),' - print '# IMPORTANT REQUIREMENT: total #writers = writers-per-node * oserver_nodes >= 2' - print '# Jobs will hang when oserver_nodes = writers-per-node = 1.' - print '#' + print (key + ':') + print () + print ('#') + print ('# OPTIONAL inputs') + print ('#') + print ('# NOTE:') + print ('# - job_name = name of experiment; default is "exp_id"') + print ('# - qos = quality-of-service; do not specify by default; specify "debug" for faster but limited service.') + print ('# - oserver_nodes = number of nodes for oserver ( default is 0 )') + print ('# - writers-per-node = tasks per oserver_node for writing ( default is 5 ),') + print ('# IMPORTANT REQUIREMENT: total #writers = writers-per-node * oserver_nodes >= 2') + print ('# Jobs will hang when oserver_nodes = writers-per-node = 1.') + print ('#') for key in optRmInpKeys: - print '#'+key + ':' + print ('#'+key + ':') def parseCmdLine(): """ @@ -1675,21 +1708,21 @@ if __name__=='__main__': args = vars(parseCmdLine()) # vars converts to dict ld = LDASsetup(args) - print "creating dir structure" + print ("creating dir structure") status = ld.createDirStructure() assert(status) - print "creating restart and bc" + print ("creating restart and bc") status = ld.createLnRstBc() assert(status) - print "creating RC Files" + print ("creating RC Files") status =ld.createRCFiles() assert status - print "creating gcm style batch Run scripts lenkf.j" + print ("creating gcm style batch Run scripts lenkf.j") status = ld.createRunScripts() - print "creating batch Run scripts" + print ("creating batch Run scripts") status = ld.createBatchRun() assert (status) diff --git a/src/Applications/LDAS_App/process_rst.csh b/src/Applications/LDAS_App/process_rst.csh deleted file mode 100755 index 677803a2..00000000 --- a/src/Applications/LDAS_App/process_rst.csh +++ /dev/null @@ -1,385 +0,0 @@ -#!/bin/csh -f - -setenv SPONSORID $1 -setenv EXPID $2 -setenv EXPDIR $3 -setenv BCSDIR $4 -setenv TILFILE $5 -setenv MODEL $6 -setenv HAVE_RESTART $7 -setenv YYYYMMDDHH $8 -setenv RESTART_ID $9 -setenv RESTART_DOMAIN $10 -setenv RESTART_PATH $11 -setenv NUMENS $12 -setenv RUN_IRRIG $13 -setenv SURFLAY $14 -setenv WEMIN_IN $15 -setenv WEMIN_OUT $16 -setenv RESTART_short ${RESTART_PATH}/${RESTART_ID}/output/${RESTART_DOMAIN}/ -setenv PARAM_FILE `ls $RESTART_short/rc_out/*/*/*ldas_catparam* | head -1` - -set PWD=`pwd` -setenv INSTDIR `echo $PWD | rev | cut -d'/' -f2- | rev` - -if ($MODEL == 'catch') then - setenv SCALE bin/Scale_Catch -else - setenv SCALE bin/Scale_CatchCN -endif - -set YYYYMMDD = `echo $YYYYMMDDHH | cut -c1-8` - -switch ($HAVE_RESTART) - -case [0] : - - echo - echo "########################################################" - echo - echo " NOTE: IMPORTANT IMPORTANT IMPROTANT " - echo " " - echo " In the absence of a restart file, " - echo " a global catch(cn)_internal_rst file is being " - echo " produced by regridding SMAP_Nature_v05 restarts " - echo " from SMAP 9km grid to the BCS grid. The start date " - echo " will be forced January 1st. " - echo " " - echo " PLEASE SPIN UP BEFORE USE " - echo - echo "########################################################" - echo - -## No restarts : regrid from archived SMAP M09 restarts - - mkdir -p $EXPDIR/$EXPID/mk_restarts/InData/ - mkdir -p $EXPDIR/$EXPID/mk_restarts/OutData/ - ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/InData/OutTileFile - ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/OutData/OutTileFile - ln -s $BCSDIR/clsm $EXPDIR/$EXPID/mk_restarts/OutData/clsm - ln -s $INSTDIR/bin $EXPDIR/$EXPID/mk_restarts/ - - cd $EXPDIR/$EXPID/mk_restarts/ - - cat << _EOI_ > mkLDASsa.j -#!/bin/csh -fx - -#SBATCH --account=${SPONSORID} -#SBATCH --time=1:00:00 -#SBATCH --ntasks=56 -#SBATCH --job-name=mkLDAS -###SBATCH --constraint=hasw -#SBATCH --qos=debug -#SBATCH --output=mkLDAS.o -#SBATCH --error=mkLDAS.e - -source $INSTDIR/bin/g5_modules -if ( -e /etc/os-release ) then - module load nco/4.8.1 -else - module load other/nco-4.6.8-gcc-5.3-sp3 -endif - -setenv LAIFILE `find ${BCSDIR}/lai_clim*` -setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/Linux/lib - -limit stacksize unlimited - -$INSTDIR/bin/esma_mpirun -np 56 bin/mk_GEOSldasRestarts -a ${SPONSORID} -b ${BCSDIR} -t ${TILFILE} -m ${MODEL} -s ${SURFLAY} -j Y - -sleep 3 - -/bin/cp InData/${MODEL}_internal_rst OutData/${MODEL}_internal_rst - -$INSTDIR/bin/esma_mpirun -np 56 bin/mk_GEOSldasRestarts -a ${SPONSORID} -b ${BCSDIR} -t ${TILFILE} -m ${MODEL} -s ${SURFLAY} -j Y - -${SCALE} InData/${MODEL}_internal_rst OutData/${MODEL}_internal_rst ${MODEL}_internal_rst $SURFLAY $WEMIN_IN $WEMIN_OUT - -# Done creating catch*_internal_rst file - -sleep 2 - -ln -s ${MODEL}_internal_rst ${MODEL}_internal_rst.$YYYYMMDD -echo DONE > done_rst_file - -_EOI_ - - sbatch mkLDASsa.j - cd $PWD - breaksw - -case [1]: - - set coordfile=${RESTART_short}/rc_out/${RESTART_ID}.ldas_tilecoord.bin - if (-e $coordfile ) then - set ENDI=`file -b $coordfile | rev | cut -d' ' -f2 | rev` - else - echo "WARNING : no file $coordfile. DEFAULT GEOSldas restart" - set ENDI = LSB - endif - - - ## restart is from old LDAS which produce big endian binary - if($ENDI == MSB) then - echo ' ' - mkdir -p $EXPDIR/$EXPID/mk_restarts/OutData/ - ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/OutData/OutTileFile - ln -s $BCSDIR/clsm $EXPDIR/$EXPID/mk_restarts/OutData/clsm - ln -s $INSTDIR/bin $EXPDIR/$EXPID/mk_restarts/ - - cd $EXPDIR/$EXPID/mk_restarts/ - echo '#\!/bin/csh -f ' > this.file - - echo "#SBATCH --account=${SPONSORID}">> this.file - echo "#SBATCH --time=1:00:00" >> this.file - echo "#SBATCH --ntasks=$NUMENS" >> this.file - echo "#SBATCH --job-name=mkRst" >> this.file - echo "#SBATCH --qos=debug" >> this.file - echo "#SBATCH --output=mkRst.o" >> this.file - echo "#SBATCH --error=mkRst.e" >> this.file - echo 'source $INSTDIR/bin/g5_modules' >> this.file - echo 'if ( -e /etc/os-release ) then' >> this.file - echo ' module load nco/4.8.1' >> this.file - echo 'else' >> this.file - echo ' module load other/nco-4.6.8-gcc-5.3-sp3 ' >> this.file - echo 'endif' >> this.file - echo 'setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/Linux/lib' >> this.file - - set mpi_mpmd = "${INSTDIR}/bin/esma_mpirun -np 1 bin/mk_GEOSldasRestarts -b ${BCSDIR} -d ${YYYYMMDDHH} -e ${RESTART_ID} -k 0000 -l ${RESTART_short} -m ${MODEL} -s ${SURFLAY} -r Y -t ${TILFILE}" - set j = 1 - while ($j < $NUMENS) - set ENS = `printf '%04d' $j` - set mpi_mpmd = "${mpi_mpmd} : -np 1 bin/mk_GEOSldasRestarts -b ${BCSDIR} -d ${YYYYMMDDHH} -e ${RESTART_ID} -k ${ENS} -l ${RESTART_short} -m ${MODEL} -s ${SURFLAY} -r Y -t ${TILFILE}" - @ j++ - end - echo $mpi_mpmd >> this.file - - set j = 0 - while ($j < $NUMENS) - set ENS = `printf '%04d' $j` - @ j++ - end - echo 'wait' >> this.file - echo 'echo DONE > done_rst_file' >> this.file - sbatch this.file - cd $PWD - sleep 1 - else - ## restart is from GEOSldas little endian binary - if( ! -e $EXPDIR/$EXPID/mk_restarts ) mkdir -p $EXPDIR/$EXPID/mk_restarts/ - echo DONE > $EXPDIR/$EXPID/mk_restarts/done_rst_file - sleep 1 - endif - - breaksw - -case [2]: - - echo ' ' - mkdir -p $EXPDIR/$EXPID/mk_restarts/InData/ - mkdir -p $EXPDIR/$EXPID/mk_restarts/OutData/ - ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/InData/OutTileFile - ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/OutData/OutTileFile - ln -s $BCSDIR/clsm $EXPDIR/$EXPID/mk_restarts/OutData/clsm - ln -s $INSTDIR/bin $EXPDIR/$EXPID/mk_restarts/ - - cd $EXPDIR/$EXPID/mk_restarts/ - - cat << _EOI3_ > mkLDASsa.j -#!/bin/csh -fx - -#SBATCH --account=${SPONSORID} -#SBATCH --time=1:00:00 -#SBATCH --ntasks=56 -#SBATCH --job-name=mkLDAS -#SBATCH --constraint=hasw -#SBATCH --qos=debug -#SBATCH --output=mkLDAS.o -#SBATCH --error=mkLDAS.e - -source $INSTDIR/bin/g5_modules -setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 -if ( -e /etc/os-release ) then - module load nco/4.8.1 -else - module load other/nco-4.6.8-gcc-5.3-sp3 -endif -setenv LAIFILE `find ${BCSDIR}/lai_clim*` -limit stacksize unlimited - -$INSTDIR/bin/esma_mpirun -np 56 bin/mk_GEOSldasRestarts -b ${BCSDIR} -d ${YYYYMMDDHH} -e ${RESTART_ID} -l ${RESTART_short} -t ${TILFILE} -m ${MODEL} -s $SURFLAY -j Y -r R -p ${PARAM_FILE} -sleep 3 - - -${SCALE} InData/${MODEL}_internal_rst OutData/${MODEL}_internal_rst ${MODEL}_internal_rst $SURFLAY $WEMIN_IN $WEMIN_OUT - -# Done creating catch*_internal_rst file - -sleep 2 - -ln -s ${MODEL}_internal_rst ${MODEL}_internal_rst.$YYYYMMDD -echo DONE > done_rst_file - -_EOI3_ - - sbatch mkLDASsa.j - cd $PWD - breaksw - -case [FGM]: - - set YYYY = `echo $YYYYMMDD | cut -c1-4` - set MM = `echo $YYYYMMDD | cut -c5-6` - set DD = `echo $YYYYMMDD | cut -c7-8` - - mkdir -p $EXPDIR/$EXPID/mk_restarts/InData/ - mkdir -p $EXPDIR/$EXPID/mk_restarts/OutData.1/ - mkdir -p $EXPDIR/$EXPID/mk_restarts/OutData.2/ - - if ($HAVE_RESTART == M) then - set ARCDIR = /archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/d5124_m2_jan79/rs/Y ; set mlable = jan79 - if ($YYYY > 1991) set ARCDIR = /archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/d5124_m2_jan91/rs/Y ; set mlable = jan91 - if ($YYYY > 2000) set ARCDIR = /archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/d5124_m2_jan00/rs/Y ; set mlable = jan00 - if ($YYYY > 2010) set ARCDIR = /archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/d5124_m2_jan10/rs/Y ; set mlable = jan10 - set rstfile = ${ARCDIR}${YYYY}/M${MM}/d5124_m2_${mlable}.${MODEL}_internal_rst.${YYYYMMDD}_21z.bin - dmget $rstfile - set INTILFILE = /gpfsm/dnb02/ltakacs/bcs/Ganymed-4_0/Ganymed-4_0_MERRA-2/CF0180x6C_DE1440xPE0720/CF0180x6C_DE1440xPE0720-Pfafstetter.til - set WEMIN_IN = 26 - /bin/cp -p $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - - if ($HAVE_RESTART == G) then - set rstfile = `echo $RESTART_PATH | rev | cut -c 2- | rev` - set INTILFILE = `readlink $RESTART_ID/scratch/tile.data` - if ( `echo $INTILFILE | grep -n 'NLv3'` == '' ) set WEMIN_IN = 26 - /bin/cp -p $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - - if ($HAVE_RESTART == F) then - - set date_16 = `date -d"2017-1-24" +%Y%m%d` - set date_17 = `date -d"2017-11-1" +%Y%m%d` - set date_21 = `date -d"2018-7-11" +%Y%m%d` - set date_22 = `date -d"2019-3-13" +%Y%m%d` - set date_25 = `date -d"2020-1-30" +%Y%m%d` - set expdate = `date -d"$YYYY-$MM-$DD" +%Y%m%d` - - if ($expdate < $date_16) then - echo "WARNING : FP restarts before $date_16 are not availale." - echo " Please select RESTART: M and use MERRA-2, instead." - exit - endif - - if (($expdate >= $date_16) && ($expdate < $date_17)) then - set fpver = GEOS-5.16/GEOSadas-5_16/ - set fplab = f516_fp - set INTILFILE = /discover/nobackup/ltakacs/bcs/Ganymed-4_0/Ganymed-4_0_Ostia/CF0720x6C_DE2880xPE1440/CF0720x6C_DE2880xPE1440-Pfafstetter.til - set WEMIN_IN = 26 - set rstfile = /archive/u/dao_ops/$fpver/${fplab}/rs/Y${YYYY}/M${MM}/${fplab}.${MODEL}_internal_rst.${YYYYMMDD}_21z.bin - dmget $rstfile - /bin/cp -p $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - if (($expdate >= $date_17) && ($expdate < $date_21)) then - set fpver = GEOS-5.17/GEOSadas-5_17/ - set fplab = f517_fp - set INTILFILE = /discover/nobackup/ltakacs/bcs/Icarus/Icarus_Ostia/CF0720x6C_CF0720x6C/CF0720x6C_CF0720x6C-Pfafstetter.til - set WEMIN_IN = 26 - set TARFILE = /archive/u/dao_ops/$fpver/${fplab}/rs/Y${YYYY}/M${MM}/${fplab}.rst.${YYYYMMDD}_21z.tar - dmget $TARFILE - set rstfile = ${fplab}.${MODEL}_internal_rst.${YYYYMMDD}_21z.nc4 - tar -xvf $TARFILE $rstfile && /bin/mv $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - if (($expdate >= $date_21) && ($expdate < $date_22)) then - set fpver = GEOS-5.21/GEOSadas-5_21/ - set fplab = f521_fp - set INTILFILE = /discover/nobackup/ltakacs/bcs/Icarus/Icarus_Ostia/CF0720x6C_CF0720x6C/CF0720x6C_CF0720x6C-Pfafstetter.til - set WEMIN_IN = 26 - set TARFILE = /archive/u/dao_ops/$fpver/${fplab}/rs/Y${YYYY}/M${MM}/${fplab}.rst.${YYYYMMDD}_21z.tar - dmget $TARFILE - set rstfile = ${fplab}.${MODEL}_internal_rst.${YYYYMMDD}_21z.nc4 - tar -xvf $TARFILE $rstfile && /bin/mv $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - if (($expdate >= $date_22) && ($expdate < $date_25)) then - set fpver = GEOS-5.22/GEOSadas-5_22/ - set fplab = f522_fpp - set INTILFILE = /discover/nobackup/ltakacs/bcs/Icarus/Icarus_Ostia/CF0720x6C_CF0720x6C/CF0720x6C_CF0720x6C-Pfafstetter.til - set WEMIN_IN = 26 - set TARFILE = /archive/u/dao_ops/$fpver/${fplab}/rs/Y${YYYY}/M${MM}/${fplab}.rst.${YYYYMMDD}_21z.tar - dmget $TARFILE - set rstfile = ${fplab}.${MODEL}_internal_rst.${YYYYMMDD}_21z.nc4 - tar -xvf $TARFILE $rstfile && /bin/mv $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - if ($expdate >= $date_25) then - set fpver = GEOS-5.25/GEOSadas-5_25/ - set fplab = f525land_fpp - set INTILFILE = /discover/nobackup/ltakacs/bcs/Icarus-NLv3/Icarus-NLv3_Ostia/CF0720x6C_CF0720x6C/CF0720x6C_CF0720x6C-Pfafstetter.til - set WEMIN_IN = 13 - set TARFILE = /archive/u/dao_ops/$fpver/${fplab}/rs/Y${YYYY}/M${MM}/${fplab}.rst.${YYYYMMDD}_21z.tar - dmget $TARFILE - set rstfile = ${fplab}.${MODEL}_internal_rst.${YYYYMMDD}_21z.nc4 - tar -xvf $TARFILE $rstfile && /bin/mv $rstfile $EXPDIR/$EXPID/mk_restarts/InData/M2Restart - endif - - endif - - /bin/ln -s $INTILFILE $EXPDIR/$EXPID/mk_restarts/InData/InTilFile - /bin/ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/OutData.1/OutTilFile - /bin/ln -s $BCSDIR/$TILFILE $EXPDIR/$EXPID/mk_restarts/OutData.2/OutTilFile - /bin/ln -s $BCSDIR/clsm $EXPDIR/$EXPID/mk_restarts/OutData.2/clsm - /bin/ln -s $INSTDIR/bin $EXPDIR/$EXPID/mk_restarts/ - cd $EXPDIR/$EXPID/mk_restarts/ - - cat << _EOI5_ > mkLDASsa.j -#!/bin/csh -fx - -#SBATCH --account=${SPONSORID} -#SBATCH --time=1:00:00 -#SBATCH --ntasks=56 -#SBATCH --job-name=mkLDAS -#SBATCH --constraint=hasw -#SBATCH --qos=debug -#SBATCH --output=mkLDAS.o -#SBATCH --error=mkLDAS.e - -source $INSTDIR/bin/g5_modules -setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 -#setenv MKL_CBWR SSE4_2 # ensure zero-diff across archs -#setenv MV2_ON_DEMAND_THRESHOLD 8192 # MVAPICH2 -setenv LAIFILE `find ${BCSDIR}/lai_clim*` -if ( -e /etc/os-release ) then - module load nco/4.8.1 -else - module load other/nco-4.6.8-gcc-5.3-sp3 -endif -limit stacksize unlimited - -/bin/ln -s OutData.1 OutData -if(${MODEL} == 'catch') then -$INSTDIR/bin/esma_mpirun -np 56 bin/mk_CatchRestarts OutData/OutTilFile InData/InTilFile InData/M2Restart $SURFLAY 4 -else -$INSTDIR/bin/esma_mpirun -np 56 bin/mk_CatchCNRestarts OutData/OutTilFile InData/InTilFile InData/M2Restart $SURFLAY $YYYYMMDD 4 -endif -/bin/rm OutData - -/bin/ln -s OutData.2 OutData -if(${MODEL} == 'catch') then -$INSTDIR/bin/esma_mpirun -np 1 bin/mk_CatchRestarts OutData/OutTilFile OutData.1/OutTilFile OutData.1/M2Restart $SURFLAY 4 -else -$INSTDIR/bin/esma_mpirun -np 1 bin/mk_CatchCNRestarts OutData/OutTilFile OutData.1/OutTilFile OutData.1/M2Restart $SURFLAY $YYYYMMDD 4 -endif -/bin/rm OutData - -${SCALE} OutData.1/M2Restart OutData.2/M2Restart ${MODEL}_internal_rst $SURFLAY 26 $WEMIN_OUT - -/bin/ln -s ${MODEL}_internal_rst ${MODEL}_internal_rst.$YYYYMMDD - -echo DONE > done_rst_file -_EOI5_ - - sbatch mkLDASsa.j - cd $PWD - breaksw - -default : - echo $HAVE_RESTART is not implemented -endsw diff --git a/src/Applications/LDAS_App/process_rst.py b/src/Applications/LDAS_App/process_rst.py new file mode 100644 index 00000000..af67bfdf --- /dev/null +++ b/src/Applications/LDAS_App/process_rst.py @@ -0,0 +1,91 @@ +import os +import sys +import subprocess +import shutil +import glob +import ruamel.yaml +import shlex +from remap_utils import * + +def remap_config_ldas(config, RESTART_str, RESTART_PATH, RESTART_ID): + yyyymmddhh = config['input']['shared']['yyyymmddhh'] + MODEL = config['input']['surface']['catch_model'] + out_dir = config['output']['shared']['out_dir'] + # MERRA2 + if RESTART_str == "M" : + config['input']['shared'] = merra2_expid(config['input']['shared']) + config['input']['shared']['rst_dir'] = out_dir+ '/merra2_tmp_'+ yyyymmddhh + config['input']['surface']['wemin'] = 26 + config['input']['shared']['bcs_dir'] = '/gpfsm/dnb02/ltakacs/bcs/Ganymed-4_0/Ganymed-4_0_MERRA-2/CF0180x6C_DE1440xPE0720/' + + if RESTART_str == "G" : + # WY note: it is a bad idea to overload restart_path and restart_id + config['input']['surface']['catch_tilefile'] = os.path.realpath(RESTART_ID+'scratch/tile.data') + config['input']['shared']['rst_dir'] = os.path.dirname(RESTART_PATH) + '/' + config['input']['surface']['wemin'] = 13 + if 'NLv' in config['input']['surface']['catch_tilefile'] : config['input']['surface']['wemin'] = 26 + + if RESTART_str == "F" : + date_16 = 20170124 + date_17 = 20171101 + date_21 = 20180711 + date_22 = 20190313 + date_25 = 20200130 + date_27 = 20210225 + date_29 = 20220228 + expdata = int(yyyymmddhh[0:8]) + if (expdate < date_16): + print( "WARNING : FP restarts before $date_16 are not availale.") + print( " Please select RESTART: M and use MERRA-2, instead.") + sys.exit(1) + + config['input']['shared']['bcs_dir'] = '/discover/nobackup/ltakacs/bcs/Icarus/Icarus_Ostia/CF0720x6C_CF0720x6C/' + config['input']['surface']['wemin'] = 26 + config['input']['shared']['rst_dir'] = out_dir+'/InData'+ '/' + suffix = '_21z.tar' + + if ((date_16 <= expdate) and (expdate < date_17)): + fpver = 'GEOS-5.16/GEOSadas-5_16/' + fplab = 'f516_fp' + config['input']['shared']['bcs_dir'] = '/discover/nobackup/ltakacs/bcs/Ganymed-4_0/Ganymed-4_0_Ostia/CF0720x6C_DE2880xPE1440/' + suffix = '_21z.bin' + + if ((date_17 <= expdate) and (expdate < date_21)): + fpver = 'GEOS-5.17/GEOSadas-5_17/' + fplab = 'f517_fp' + + if ((date_21 <= expdate) and (expdate < date_22)): + fpver = 'GEOS-5.21/GEOSadas-5_21/' + fplab = 'f521_fp' + + if ((date_22 <= expdate) and (expdate < date_25)): + fpver = 'GEOS-5.22/GEOSadas-5_22/' + fplab = 'f522_fp' + + if (date_25 <= expdate and expdate < date_27) : + fpver = 'GEOS-5.25/GEOSadas-5_25/' + fplab = 'f525land_fpp' + config['input']['surface']['wemin'] = 13 + + if (date_27 <= expdate and expdate < date_29) : + fpver = 'GEOS-5.27/GEOSadas-5_27/' + fplab = 'f5271_fpp' + config['input']['surface']['wemin'] = 13 + + if (date_29 <= expdate ) : + fpver = 'GEOS-5.29/GEOSadas-5_29/' + fplab = 'f5293_fpp' + config['input']['surface']['wemin'] = 13 + + + rstfile = '/archive/u/dao_ops/'+fpver+'/'+fplab+'/rs/Y'+YYYY+'/M'+MM+'/'+fplab+'.'+MODEL+'_internal_rst.'+YYYYMMDD + suffix + fname = os.path.basename(rstfile) + dest = out_dir+'/InData'+ '/'+fname + print("Copy file "+ rstfile +" to " + out_dir+'/InData'+ '/') + shutil.copy(rstfile, dest) + + if suffix == "_21z.tar" : + new_rst = out_dir+'/InData'+ '/'+ fplab+'.'+MODEL+'_internal_rst.'+YYYYMMDD+'_21z.nc4' + subprocess.call(['tar', '-xvf', dest, new_rst]) + + return config