Skip to content

Commit

Permalink
Merge branch 'develop' into release/MAPL-v3
Browse files Browse the repository at this point in the history
  • Loading branch information
mathomp4 committed Jan 6, 2025
2 parents 2f6f781 + 65b0002 commit ae0361b
Show file tree
Hide file tree
Showing 3 changed files with 226 additions and 0 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- Added utility to prepare inputs for ExtDatDriver.x so that ExtData can simulate a real GEOS run
- Added loggers when writing or reading weight files
- Added new option to AGCM.rc `overwrite_checkpoint` to allow checkpoint files to be overwritten. By default still will not overwrite checkpoints

Expand Down
58 changes: 58 additions & 0 deletions Tests/generate_extdatadriver_input.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Introduction
This is a simple utility to generate inputs for ExtDataDriver.x so that ExtData can simulate a real GEOS run. It requires 3 things that are passed in
- A list of items ExtData needs to fill. This can be found by looking at the GEOS log
- The import spec of the GCM component from using the printspec option to GEOS
- all the needed yaml files in a directory, name of directory is passed

This will generate the import and optionally the export (as well as History) defition for ExtDataDriver.x to spare the human the tedious work.

# Example Inputs
To get the list of ExtDate items, just grab all the lines that look like this to a file:
```
EXTDATA: INFO: ---- 00001: BC_AIRCRAFT
EXTDATA: INFO: ---- 00002: BC_ANTEBC1
EXTDATA: INFO: ---- 00003: BC_ANTEBC2
EXTDATA: INFO: ---- 00004: BC_AVIATION_CDS
EXTDATA: INFO: ---- 00005: BC_AVIATION_CRS
EXTDATA: INFO: ---- 00006: BC_AVIATION_LTO
EXTDATA: INFO: ---- 00007: BC_BIOFUEL
EXTDATA: INFO: ---- 00008: BC_BIOMASS
EXTDATA: INFO: ---- 00009: BC_SHIP
EXTDATA: INFO: ---- 00010: BRC_AIRCRAFT
EXTDATA: INFO: ---- 00011: BRC_ANTEBRC1
EXTDATA: INFO: ---- 00012: BRC_ANTEBRC2
EXTDATA: INFO: ---- 00013: BRC_AVIATION_CDS
EXTDATA: INFO: ---- 00014: BRC_AVIATION_CRS
EXTDATA: INFO: ---- 00015: BRC_AVIATION_LTO
EXTDATA: INFO: ---- 00016: BRC_BIOFUEL
EXTDATA: INFO: ---- 00017: BRC_BIOMASS
EXTDATA: INFO: ---- 00018: BRC_SHIP
EXTDATA: INFO: ---- 00019: BRC_TERPENE
```

To get the GCM component spec, run with `PRINTSPEC: 1` in the `CAP.rc` and copy lines out that look like this:
```
#IMPORT spec for GCM
#COMPONENT, SHORT_NAME, LONG_NAME, UNIT, DIMS, CONTAINER_TYPE
GENERIC: INFO: GCM, WSUB_CLIM, stdev in vertical velocity, m s-1, 3, esmf_field
GENERIC: INFO: GCM, MEGAN_ORVC, MEGAN_ORVC, kgC/m2/s, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_CROP, CLM4_PFT_CROP, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_C4_GRSS, CLM4_PFT_C4_GRSS, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_C3_NARC_GRSS, CLM4_PFT_C3_NARC_GRSS, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_C3_ARCT_GRSS, CLM4_PFT_C3_ARCT_GRSS, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_DECD_BORL_SHRB, CLM4_PFT_BDLF_DECD_BORL_SHRB, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_DECD_TMPT_SHRB, CLM4_PFT_BDLF_DECD_TMPT_SHRB, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_EVGN_SHRB, CLM4_PFT_BDLF_EVGN_SHRB, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_DECD_BORL_TREE, CLM4_PFT_BDLF_DECD_BORL_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_DECD_TMPT_TREE, CLM4_PFT_BDLF_DECD_TMPT_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_DECD_TROP_TREE, CLM4_PFT_BDLF_DECD_TROP_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_EVGN_TMPT_TREE, CLM4_PFT_BDLF_EVGN_TMPT_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_BDLF_EVGN_TROP_TREE, CLM4_PFT_BDLF_EVGN_TROP_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_NDLF_DECD_BORL_TREE, CLM4_PFT_NDLF_DECD_BORL_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_NDLF_EVGN_BORL_TREE, CLM4_PFT_NDLF_EVGN_BORL_TREE, 1, 2, esmf_field
GENERIC: INFO: GCM, CLM4_PFT_NDLF_EVGN_TMPT_TREE, CLM4_PFT_NDLF_EVGN_TMPT_TREE, 1, 2, esmf_field
```

Finally just grab the right yaml files for ExtData.

To run you will of course need to do some further editing of the produced files and link in the acutal data using the same convention `gcm_run.j` does.
167 changes: 167 additions & 0 deletions Tests/generate_extdatadriver_input.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
from yaml import load,dump
import argparse
import os
import yaml
import glob

dims_dict = {"2":"xy", "3":"xyz"}

def get_dims(component_map, name):
for comp in component_map:
if name in component_map[comp]:
dims = component_map[comp][name]["dims"]
return dims

def get_vars_needed(input_file):

output_list = []
f = open(input_file,"r")
lines = f.readlines()
f.close()
for line in lines:
temp = line.split()
output_list.append(temp[4])

return output_list

def get_extdata_map(input_dir):

input_files = glob.glob(input_dir+"/*.yaml")
export_list = {}
for input_file in input_files:
f = open(input_file,'r')
extdata_def = yaml.safe_load(f)
f.close()
for short_name in extdata_def["Exports"]:
export_list.update({short_name:extdata_def["Exports"][short_name]})
return export_list

def get_block(cvs_file):

temp = cvs_file[0].split(' ')
state_type = temp[1][1:].strip()
component = temp[4].strip()
i=2
for line in cvs_file[2:]:
if "spec for" in line:
break
i=i+1
return component,state_type,i


def get_component_map(cvs_file):

i_start = 0
i_end = 0
n_lines = len(cvs_file)
components = {}
while i_start < n_lines-1:

comp_name,state_type,i_end = get_block(cvs_file[i_start:])
comp_map = {}
for i in range(i_end-2):
line = cvs_file[i_start+2+i]
values = line.split(',')
short_name = values[1].strip()
long_name = values[2].strip()
units = values[3].strip()
dims = values[4].strip()
item_type = values[5].strip()
comp_map.update({short_name:{"long_name":long_name,"units":units,"item_type":item_type,"dims":dims}})
components.update({comp_name+"_"+state_type:comp_map})
i_start = i_start + i_end

return components

def parse_args():
p = argparse.ArgumentParser(description='Generarte input files for ExtDataDriver to simulate GEOS')
p.add_argument('extdata_provided',type=str,help='a list of items ExtData should fill',default=None)
p.add_argument('spec_def',type=str,help='the GEOS gcm import state from the printspec',default=None)
p.add_argument('extdata_dir',type=str,help='diretory with all the yaml imputs for extdata',default=None)
p.add_argument('-e','--export',action='store_true',help='also include exports for corresponding imports')

return vars(p.parse_args())

if __name__ == '__main__':

args = parse_args()

extdata_list = args['extdata_provided']
do_exports = args['export']
input_file = args['spec_def']
f = open(input_file,"r")
input_rc = f.readlines()
f.close()

extdata_directory = args['extdata_dir']
extdata_def = get_extdata_map(extdata_directory)

f_agcm = open("AGCM.rc",'w')
# component
component_map = {}
component_map = get_component_map(input_rc)

vars_needed = get_vars_needed(extdata_list)

nl = "\n"
cm = " , "

# Import state
written = []
f_agcm.write("IMPORT_STATE::"+nl)

for item in vars_needed:
if item in extdata_def:
long_name = "NA"
units = "NA"
dims = get_dims(component_map, item)
cdims = dims_dict[dims]

if item not in written:
f_agcm.write(item+cm+long_name+cm+units+cm+cdims+cm+"c"+nl)
written.append(item)

f_agcm.write("::"+nl)

# Export state
if do_exports:
written = []
f_agcm.write("EXPORT_STATE::"+nl)
for item in vars_needed:
if item in extdata_def:
long_name = "NA"
units = "NA"
dims = get_dims(component_map, item)
cdims = dims_dict[dims]

if item not in written:
f_agcm.write(item+cm+long_name+cm+units+cm+cdims+cm+"c"+nl)
written.append(item)

f_agcm.write("::"+nl)

f_hist = open("HISTORY.rc",'w')
f_hist.write("GRID_LABELS:"+nl)
f_hist.write("::"+nl)
f_hist.write("COLLECTIONS: my_collection"+nl)
f_hist.write("::"+nl)
f_hist.write("my_collection.template: 'nc4'"+nl)
f_hist.write("my_collection.format: 'CFIO'"+nl)
f_hist.write("my_collection.frequency: '240000'"+nl)
first = True
written = []
for item in vars_needed:
if item in extdata_def:
if item not in written:
if first:
first = False
f_hist.write("my_collection.fields:'"+item+"' , 'Root',"+"\n")
else:
f_hist.write("'"+item+"' , 'Root',"+"\n")
written.append(item)
f_hist.write("::")

f_hist.close()
f_agcm.close()

0 comments on commit ae0361b

Please sign in to comment.