From 6dce8ef96c42a790f0a5bb8c4cc87b96b204d620 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 10:56:01 -0500 Subject: [PATCH 01/45] add in cmip6-cmor-tables as a submodule, remove un-maintained gitpython pip-only dependency... nasty tech debt i should get rid of before its too late! --- .gitmodules | 3 +++ environment.yml | 7 ++----- fre/cmor/tests/test_cmor_run_subtool.py | 22 +++++++--------------- meta.yaml | 7 +++---- setup.py | 3 +-- 5 files changed, 16 insertions(+), 26 deletions(-) diff --git a/.gitmodules b/.gitmodules index b95f9e27..0196328b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "fre/gfdl_msd_schemas"] path = fre/gfdl_msd_schemas url = https://github.com/NOAA-GFDL/gfdl_msd_schemas +[submodule "fre/tests/test_files/cmip6-cmor-tables"] + path = fre/tests/test_files/cmip6-cmor-tables + url = https://github.com/PCMDI/cmip6-cmor-tables \ No newline at end of file diff --git a/environment.yml b/environment.yml index 89e18766..1683fe57 100644 --- a/environment.yml +++ b/environment.yml @@ -1,10 +1,9 @@ -name: fre-cli +name: fre-cli-test channels: - - defaults - conda-forge - noaa-gfdl dependencies: - - python + - python>=3.10 - pip - click - pyyaml @@ -23,5 +22,3 @@ dependencies: - conda-forge::pytest-cov - conda-forge::python-cdo - conda-forge::cdo>=2.0.0 - - pip: - - GitPython diff --git a/fre/cmor/tests/test_cmor_run_subtool.py b/fre/cmor/tests/test_cmor_run_subtool.py index 92fde66e..7d230cad 100644 --- a/fre/cmor/tests/test_cmor_run_subtool.py +++ b/fre/cmor/tests/test_cmor_run_subtool.py @@ -4,8 +4,6 @@ from pathlib import Path from datetime import date -import git - import fre import subprocess @@ -14,22 +12,15 @@ ROOTDIR = 'fre/tests/test_files' # setup- cmip/cmor variable table(s) -CLONE_CMIP_TABLE_URL = \ - 'https://github.com/PCMDI/cmip6-cmor-tables.git' CLONE_REPO_PATH = \ f'{ROOTDIR}/cmip6-cmor-tables' TABLE_CONFIG = \ f'{CLONE_REPO_PATH}/Tables/CMIP6_Omon.json' def test_setup_cmor_cmip_table_repo(): - ''' setup routine, if it doesnt exist, clone the repo holding CMOR/CMIP6 tables ''' - if Path(TABLE_CONFIG).exists(): - pass - else: - git.Repo.clone_from( - CLONE_CMIP_TABLE_URL, - CLONE_REPO_PATH ) - assert Path(TABLE_CONFIG).exists() + ''' setup routine, make sure the recursively cloned tables exist ''' + assert Path(TABLE_CONFIG).exists(): + assert Path(CLONE_REPO_PATH).exists() # explicit inputs to tool INDIR = f'{ROOTDIR}/ocean_sos_var_file' @@ -52,9 +43,10 @@ def test_setup_cmor_cmip_table_repo(): FULL_INPUTFILE=f"{INDIR}/{FILENAME}.nc" def test_setup_fre_cmor_run_subtool(capfd): - ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen output file from prev pytest runs,removes it if it's present, and ensures the new file is created without error. ''' - - ''' set-up test: create binary test files from reduced ascii files in root dir ''' + ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen + output file from prev pytest runs, removes it if it's present, and ensures the new file is + created without error. + ''' ncgen_input = f"{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl" ncgen_output = f"{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc" diff --git a/meta.yaml b/meta.yaml index 71c8b213..dceaaf41 100644 --- a/meta.yaml +++ b/meta.yaml @@ -14,16 +14,15 @@ build: noarch: python channels: - - defaults - conda-forge - noaa-gfdl requirements: host: - - python + - python>=3.10 - pip run: - - python + - python>=3.10 - click - pyyaml - pylint @@ -65,7 +64,7 @@ test: commands: - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." # run pytest but ignore any tests that require compilation - - pip install GitPython && pytest --ignore=fre/make/tests/compilation --config-file=fre/pytest.ini --cov-report term-missing --cov-config=fre/coveragerc --cov=fre fre/ + - pytest --ignore=fre/make/tests/compilation --config-file=fre/pytest.ini --cov-report term-missing --cov-config=fre/coveragerc --cov=fre fre/ - fre --help - fre app --help - fre catalog --help diff --git a/setup.py b/setup.py index d01e5577..9434196e 100644 --- a/setup.py +++ b/setup.py @@ -17,8 +17,7 @@ 'cylc-flow', 'cylc-rose', 'cdo', - 'metomi-rose', - 'GitPython' + 'metomi-rose' ], entry_points={ 'console_scripts': [ From f59bd8737030dec46ee5fdd957feda367d85ef0a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 11:32:24 -0500 Subject: [PATCH 02/45] i guess .gitmodules isnt meant to be edited by hand! --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0196328b..b95f9e27 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,3 @@ [submodule "fre/gfdl_msd_schemas"] path = fre/gfdl_msd_schemas url = https://github.com/NOAA-GFDL/gfdl_msd_schemas -[submodule "fre/tests/test_files/cmip6-cmor-tables"] - path = fre/tests/test_files/cmip6-cmor-tables - url = https://github.com/PCMDI/cmip6-cmor-tables \ No newline at end of file From 1168d5d38248b18e93106bdaecec09a119bfe495 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 11:34:38 -0500 Subject: [PATCH 03/45] Revert "i guess .gitmodules isnt meant to be edited by hand!" This reverts commit f59bd8737030dec46ee5fdd957feda367d85ef0a. --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index b95f9e27..0196328b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "fre/gfdl_msd_schemas"] path = fre/gfdl_msd_schemas url = https://github.com/NOAA-GFDL/gfdl_msd_schemas +[submodule "fre/tests/test_files/cmip6-cmor-tables"] + path = fre/tests/test_files/cmip6-cmor-tables + url = https://github.com/PCMDI/cmip6-cmor-tables \ No newline at end of file From a197538c30e9dbd844f6ff681df0386b071ee164 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 11:37:55 -0500 Subject: [PATCH 04/45] actually add the submodule... --- .gitmodules | 4 ++-- fre/tests/test_files/cmip6-cmor-tables | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 160000 fre/tests/test_files/cmip6-cmor-tables diff --git a/.gitmodules b/.gitmodules index 0196328b..e8f9a9c2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,5 +5,5 @@ path = fre/gfdl_msd_schemas url = https://github.com/NOAA-GFDL/gfdl_msd_schemas [submodule "fre/tests/test_files/cmip6-cmor-tables"] - path = fre/tests/test_files/cmip6-cmor-tables - url = https://github.com/PCMDI/cmip6-cmor-tables \ No newline at end of file + path = fre/tests/test_files/cmip6-cmor-tables + url = https://github.com/pcmdi/cmip6-cmor-tables diff --git a/fre/tests/test_files/cmip6-cmor-tables b/fre/tests/test_files/cmip6-cmor-tables new file mode 160000 index 00000000..a46dcbf1 --- /dev/null +++ b/fre/tests/test_files/cmip6-cmor-tables @@ -0,0 +1 @@ +Subproject commit a46dcbf17ec5e11af23dc2d55107f5e52afbcade From ff8d8ce0f8ad1c0391856ea77085cf0607a1a288 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 12:47:25 -0500 Subject: [PATCH 05/45] restore deps with the exception of the pip-installed gitpython and leaving cmor>=3.9.0 because i know we should not bother with anything less than 3.9.0 for cmor --- environment.yml | 6 +++--- meta.yaml | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index 1683fe57..0a82e938 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,9 @@ -name: fre-cli-test +name: fre_cli channels: - conda-forge - noaa-gfdl dependencies: - - python>=3.10 + - python>=3.9.0 - pip - click - pyyaml @@ -16,7 +16,7 @@ dependencies: - conda-forge::cylc-flow>=8.2.0 - conda-forge::cylc-rose - conda-forge::metomi-rose - - conda-forge::cmor + - conda-forge::cmor>=3.9.0 - conda-forge::cylc-uiserver - conda-forge::pytest - conda-forge::pytest-cov diff --git a/meta.yaml b/meta.yaml index dceaaf41..f6956cf9 100644 --- a/meta.yaml +++ b/meta.yaml @@ -19,10 +19,11 @@ channels: requirements: host: - - python>=3.10 + - python>=3.9 - pip run: - - python>=3.10 + - python>=3.9 + - pip - click - pyyaml - pylint @@ -34,7 +35,7 @@ requirements: - conda-forge::cylc-flow>=8.2.0 - conda-forge::cylc-rose - conda-forge::metomi-rose - - conda-forge::cmor + - conda-forge::cmor>=3.9.0 - conda-forge::cylc-uiserver - conda-forge::pytest - conda-forge::pytest-cov From 32abb9b403e132126b1809dd4792cce6236ca0c8 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 12:57:06 -0500 Subject: [PATCH 06/45] syntax error --- fre/cmor/tests/test_cmor_run_subtool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fre/cmor/tests/test_cmor_run_subtool.py b/fre/cmor/tests/test_cmor_run_subtool.py index 7d230cad..923b3cf6 100644 --- a/fre/cmor/tests/test_cmor_run_subtool.py +++ b/fre/cmor/tests/test_cmor_run_subtool.py @@ -19,8 +19,9 @@ def test_setup_cmor_cmip_table_repo(): ''' setup routine, make sure the recursively cloned tables exist ''' - assert Path(TABLE_CONFIG).exists(): - assert Path(CLONE_REPO_PATH).exists() + assert all( [ Path(CLONE_REPO_PATH).exists(), + Path(TABLE_CONFIG).exists() + ] ) # explicit inputs to tool INDIR = f'{ROOTDIR}/ocean_sos_var_file' From ac61693134bcede4cd3a10eb5bd8fd25ad076193 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 24 Dec 2024 13:38:25 -0500 Subject: [PATCH 07/45] whitespace-cleanup some files for lint complaints, poke the test file with the zombie-problem in the github-repos conda build ci workflow --- fre/cmor/cmor_lister.py | 28 +++---- fre/cmor/cmor_mixer.py | 98 ++++++++++++------------- fre/cmor/tests/test_cmor_run_subtool.py | 10 +-- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/fre/cmor/cmor_lister.py b/fre/cmor/cmor_lister.py index 95ebb8ec..e7cfcd1d 100644 --- a/fre/cmor/cmor_lister.py +++ b/fre/cmor/cmor_lister.py @@ -1,4 +1,4 @@ -''' fre cmor list +''' fre cmor list because ian got tired of typing things like the following in bash... varname=sos; \ @@ -29,7 +29,7 @@ def print_var_content( table_config_file = None, var_name = None): proj_table_vars=json.load(table_config_file) except Exception as exc: raise Exception(f'problem getting proj_table_vars... WHY') - + var_content = None try: var_content = proj_table_vars["variable_entry"].get(var_name) @@ -41,7 +41,7 @@ def print_var_content( table_config_file = None, var_name = None): if var_content is None: #print(f'(cmor_list_subtool) variable {var_name} not found in {Path(json_table_config).name}, moving on!') return - + table_name = None try: #print(f'(print_var_content) trying to get table_name from proj_table_vars...') @@ -61,14 +61,14 @@ def print_var_content( table_config_file = None, var_name = None): for content in var_content: if content in DO_NOT_PRINT_LIST: continue - print(f' {content}: {var_content[content]}') + print(f' {content}: {var_content[content]}') print('\n') - + return def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): - ''' - finds tables in the CMIP json config directory containing variable data of interest. prints it + ''' + finds tables in the CMIP json config directory containing variable data of interest. prints it out to screen, intended largely as a helper tool for cli users. ''' if not Path(json_table_config_dir).exists(): @@ -85,30 +85,30 @@ def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_v if json_var_list is not None: with open( json_var_list, "r", encoding = "utf-8") as var_list_file : var_list=json.load(var_list_file) - + if opt_var_name is None and var_list is None: raise ValueError(f'(cmor_list_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') if opt_var_name is not None: print(f'(cmor_list_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') for json_table_config in json_table_configs: - #print(f'(cmor_list_subtool) attempting to open {json_table_config}') + #print(f'(cmor_list_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: print_var_content(table_config_file, opt_var_name) - + elif var_list is not None: print(f'(cmor_list_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') - for var in var_list: + for var in var_list: for json_table_config in json_table_configs: - #print(f'(cmor_list_subtool) attempting to open {json_table_config}') + #print(f'(cmor_list_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: #print(f' var = {var}, var_list[{var}]={var_list[var]}') print_var_content(table_config_file, str(var_list[var])) else: print(f'(FATAL) this line should be unreachable!!!') - + return - + @click.command() def _cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 1047ba75..c561de71 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -33,7 +33,7 @@ def from_dis_gimme_dis(from_dis, gimme_dis): except Exception as exc: print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' # f' from this: {from_dis} ' - f' exc = {exc}' + f' exc = {exc}' f' returning None!' ) return None @@ -53,18 +53,18 @@ def find_statics_file(bronx_file_path): return statics_file else: return None - + def create_lev_bnds(bound_these = None, with_these = None): - the_bnds = None + the_bnds = None assert len(with_these) == len(bound_these) + 1 print(f'(create_lev_bnds) bound_these is... ') print(f' bound_these = \n{bound_these}') print(f'(create_lev_bnds) with_these is... ') print(f' with_these = \n{with_these}') - - the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) + + the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) for i in range(0,len(bound_these)): the_bnds[i][0]=with_these[i] the_bnds[i][1]=with_these[i+1] @@ -128,7 +128,7 @@ def check_dataset_for_ocean_grid(ds): " sometimes i don't cmorize right! check me!") return True return False - + def get_vertical_dimension(ds, target_var): @@ -141,7 +141,7 @@ def get_vertical_dimension(ds, target_var): vert_dim = 0 for name, variable in ds.variables.items(): if name != target_var: # not the var we are looking for? move on. - continue + continue dims = variable.dimensions for dim in dims: #print(f'(get_vertical_dimension) dim={dim}') @@ -154,7 +154,7 @@ def get_vertical_dimension(ds, target_var): if not (ds[dim].axis and ds[dim].axis == "Z"): continue vert_dim = dim - + return vert_dim def create_tmp_dir(outdir, json_exp_config = None): @@ -189,7 +189,7 @@ def create_tmp_dir(outdir, json_exp_config = None): # once we know where the tmp_dir should be, create it try: os.makedirs(tmp_dir, exist_ok=True) - # and if we need to additionally create outdir_from_exp_config... try doing that too + # and if we need to additionally create outdir_from_exp_config... try doing that too if outdir_from_exp_config is not None: print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') try: @@ -213,7 +213,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, json_exp_config = None, json_table_config = None, prev_path=None, ):#, tmp_dir = None ): - ''' + ''' rewrite the input netcdf file nc_fl containing target_var in a CMIP-compliant manner. accepts six arguments, all required: proj_table_vars: json dictionary object, variable table read from json_table_config. @@ -224,7 +224,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, json_exp_config: string, representing path to json configuration file holding metadata for appending to output this argument is most used for making sure the right grid label is getting attached to the right output json_table_config: string, representing path to json configuration file holding variable names for a given table. - proj_table_vars is read from this file, but both are passed anyways. + proj_table_vars is read from this file, but both are passed anyways. ''' print('\n\n-------------------------- START rewrite_netcdf_file_var call -----') print( "(rewrite_netcdf_file_var) input data: " ) @@ -242,18 +242,18 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, uses_ocean_grid = check_dataset_for_ocean_grid(ds) if uses_ocean_grid: print('(rewrite_netcdf_file_var) OH BOY you have a file on the native tripolar grid...\n' - ' ... this is gonna be fun!' ) + ' ... this is gonna be fun!' ) # try to read what coordinate(s) we're going to be expecting for the variable expected_mip_coord_dims=None try: expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"] print( '(rewrite_netcdf_file_var) i am hoping to find data for the following coordinate dimensions:\n' - f' expected_mip_coord_dims = {expected_mip_coord_dims}' ) + f' expected_mip_coord_dims = {expected_mip_coord_dims}' ) except Exception as exc: print(f'(rewrite_netcdf_file_var) WARNING could not get expected coordinate dimensions for {target_var}. ' ' in proj_table_vars file {json_table_config}. \n exc = {exc}') - + ## figure out the coordinate/dimension names programmatically TODO @@ -282,22 +282,22 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # read in time_bnds , if present print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') time_bnds = from_dis_gimme_dis( from_dis = ds, - gimme_dis = 'time_bnds' ) + gimme_dis = 'time_bnds' ) # read the input variable data, i believe print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}') var = from_dis_gimme_dis( from_dis = ds, - gimme_dis = target_var ) + gimme_dis = target_var ) #var = ds[target_var][:] # the tripolar grid is designed to reduce distortions in ocean data brought on # by singularities (poles) being placed in oceans (e.g. the N+S poles of standard sphere grid) # but, the tripolar grid is complex, so the values stored in the file are a lat/lon *on the tripolar grid* # in order to get spherical lat/lon, one would need to convert on the fly, but implementing such an inverse is not trivial # thankfully, the spherical lat/lons tend to already be computed in advance, and stored elsewhere. at GFDL they're in "statics" - do_special_ocean_file_stuff=all( [ uses_ocean_grid, - lat is None, + do_special_ocean_file_stuff=all( [ uses_ocean_grid, + lat is None, lon is None ] ) - + statics_file_path = None x, y = None, None i_ind, j_ind = None, None @@ -328,9 +328,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'BAR min entry of geolon: {statics_lon[:].data.min()}') lat = ds.createVariable('lat', np.float32, ('yh', 'xh') ) - lat[:] = statics_lat[:] + lat[:] = statics_lat[:] lon = ds.createVariable('lon', np.float32, ('yh', 'xh') ) - lon[:] = statics_lon[:] + lon[:] = statics_lon[:] print(f'FOO min entry of lat: {lat[:].data.min()}') print(f'BAR min entry of lon: {lon[:].data.min()}') @@ -365,7 +365,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, j_ind = ds.createVariable('j_index', np.int32, ('yh') ) print(f' np.arange...') #j_ind[:] = np.zeros(len(y), dtype=int ) - j_ind[:] = np.arange(0, len(y), dtype=np.int32 ) + j_ind[:] = np.arange(0, len(y), dtype=np.int32 ) print(f'(rewrite_netcdf_file_var) HARD PART: creating indices (i_index) from x (xh)') @@ -383,23 +383,23 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, #var.coordinates = 'lat lon' var.coordinates = 'j_index i_index' #var.coordinates = '' - - - - - - + + + + + + #print(f' geolat = {lat}') #assert False - - - - + + + + # grab var_dim @@ -491,27 +491,27 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, latitude = lat[:], longitude = lon[:], latitude_vertices = lat_bnds[:], longitude_vertices = lon_bnds[:]) - + # load back up the normal table file? - cmor.load_table(json_table_config) + cmor.load_table(json_table_config) # setup cmor time axis if relevant cmor_time = None print(f'(rewrite_netcdf_file_var) assigning cmor_time') - try: #if vert_dim != 'landuse': + try: #if vert_dim != 'landuse': print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" f" cell_bounds = time_bnds, units = {time_coord_units}) ") print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, cell_bounds = time_bnds, units = time_coord_units) - except ValueError as exc: #else: + except ValueError as exc: #else: print(f"(rewrite_netcdf_file_var) cmor_time = cmor.axis('time', \n" " coord_vals = time_coords, units = time_coord_units)") - print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') + print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) print(f' DONE assigning cmor_time') - + # # setup cmor time axis if relevant # cmor_time = None # try: @@ -525,11 +525,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n" # " cmor_time = cmor.axis('time', \n" # " coord_vals = time_coords, units = time_coord_units)") -# print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') -# cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - +# print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') +# cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) + + - # other vertical-axis-relevant initializations save_ps = False ps = None @@ -615,21 +615,21 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, axis_ids.append(cmor_time) print(f' axis_ids now = {axis_ids}') if cmor_lat is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lat to axis_ids list...') + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axis_ids list...') axis_ids.append(cmor_lat) print(f' axis_ids now = {axis_ids}') if cmor_lon is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lon to axis_ids list...') + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axis_ids list...') axis_ids.append(cmor_lon) print(f' axis_ids now = {axis_ids}') - + ips = cmor.zfactor( zaxis_id = cmor_lev, zfactor_name = "ps", axis_ids = axis_ids, #[cmor_time, cmor_lat, cmor_lon], units = "Pa" ) save_ps = True print(f' DONE assigning cmor_lev') - + axes = [] if cmor_time is not None: @@ -637,15 +637,15 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, axes.append(cmor_time) print(f' axes now = {axes}') if cmor_lev is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lev to axes list...') + print(f'(rewrite_netcdf_file_var) appending cmor_lev to axes list...') axes.append(cmor_lev) print(f' axes now = {axes}') if cmor_lat is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lat to axes list...') + print(f'(rewrite_netcdf_file_var) appending cmor_lat to axes list...') axes.append(cmor_lat) print(f' axes now = {axes}') if cmor_lon is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lon to axes list...') + print(f'(rewrite_netcdf_file_var) appending cmor_lon to axes list...') axes.append(cmor_lon) print(f' axes now = {axes}') diff --git a/fre/cmor/tests/test_cmor_run_subtool.py b/fre/cmor/tests/test_cmor_run_subtool.py index 923b3cf6..11ecb018 100644 --- a/fre/cmor/tests/test_cmor_run_subtool.py +++ b/fre/cmor/tests/test_cmor_run_subtool.py @@ -44,16 +44,16 @@ def test_setup_cmor_cmip_table_repo(): FULL_INPUTFILE=f"{INDIR}/{FILENAME}.nc" def test_setup_fre_cmor_run_subtool(capfd): - ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen - output file from prev pytest runs, removes it if it's present, and ensures the new file is - created without error. + ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen + output file from prev pytest runs, removes it if it's present, and ensures the new file is + created without error. ''' ncgen_input = f"{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl" ncgen_output = f"{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc" if Path(ncgen_output).exists(): - Path(ncgen_output).unlink() + Path(ncgen_output).unlink() assert Path(ncgen_input).exists() ex = [ 'ncgen3', '-k', 'netCDF-4', '-o', ncgen_output, ncgen_input ] @@ -112,7 +112,7 @@ def test_fre_cmor_run_subtool_case1_output_compare_data(capfd): check=False, capture_output=True ) - + # err_list has length two if end in newline err_list = result.stderr.decode().split('\n') expected_err = \ From 91a0fbc9612e765f3ea1f5c9c82c88056d29f022 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 3 Jan 2025 09:15:15 -0500 Subject: [PATCH 08/45] add pyproject.toml to suppress pip warning. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7ad0c21c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=64", "setuptools-scm>=8"] +build-backend = "setuptools.build_meta" From cec2a128d02bd814bb802a4bbbbfa6345ab23f67 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 3 Jan 2025 13:53:26 -0500 Subject: [PATCH 09/45] trying to get a fre make test's output out of the way, getting surprisingly pathological behavior. commit. wipe dir. re-clone and rebuild for my sanity's sake... --- .../compilation/test_run_fremake_builds.py | 2 ++ fre/tests/__init__.py | 0 fre/tests/test_fre_make_cli.py | 29 +++++++++++++------ pyproject.toml | 3 -- 4 files changed, 22 insertions(+), 12 deletions(-) delete mode 100644 fre/tests/__init__.py delete mode 100644 pyproject.toml diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 9fe6169e..1be4731c 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -31,6 +31,7 @@ # test building the null model using gnu compilers +@pytest.mark.skip(reason="EXPECTED TO FAIL ON WORKSTATION REMOVE THIS LINE PREMERGE") def test_run_fremake_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH @@ -38,6 +39,7 @@ def test_run_fremake_serial_compile(): assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() # same test with a parallel build +@pytest.mark.skip(reason="EXPECTED TO FAIL ON WORKSTATION REMOVE THIS LINE PREMERGE") def test_run_fremake_multijob_compile(): ''' test run-fremake parallel compile with gnu''' os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH diff --git a/fre/tests/__init__.py b/fre/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 279760b3..5e75307d 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -3,6 +3,7 @@ from click.testing import CliRunner from pathlib import Path import os +import shutil from fre import fre runner = CliRunner() @@ -22,33 +23,37 @@ def test_cli_fre_make_opt_dne(): result = runner.invoke(fre.fre, args=["make", "optionDNE"]) assert result.exit_code == 2 +TEST_DIR = Path("fre/tests") +OUT_PATH=f"{TEST_DIR}/test_files/fremake_out" def test_cli_fre_make_create_checkout_baremetal(): ''' fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t debug''' + # remove the created script to re-create it, if it exists + #if Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").exists(): + # Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").unlink() + # Set paths and click options - test_dir = Path("fre/tests") yamlfile = Path("fre/make/tests/AM5_example/") platform = "ncrc5.intel23" target = "debug" # Create output path to test that files exist - out_path=f"{test_dir}/fremake_out" - Path(out_path).mkdir(parents=True,exist_ok=True) + Path(OUT_PATH).mkdir(parents=True,exist_ok=True) # Set HOME for modelRoot location (output location) in fre make - os.environ["HOME"]=str(Path(out_path)) + os.environ["HOME"]=str(Path(OUT_PATH)) # run create-checkout result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) - # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) + # Check for successful command, creation of checkout script, and that script is executable + # os.access - checks is file has specific access mode, os.X_OK - checks executable permission assert all ([result.exit_code == 0, - Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh").exists(), - os.access(Path(f"{out_path}/fremake_canopy/test/am5/src/checkout.sh"), os.X_OK)]) + Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").exists(), + os.access(Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh"), os.X_OK)]) def test_cli_fre_make_create_checkout_container(): ''' fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug''' # Set paths and click options - test_dir = Path("fre/tests") yamlfile = Path("fre/make/tests/AM5_example/") platform = "hpcme.2023" target = "debug" @@ -56,7 +61,13 @@ def test_cli_fre_make_create_checkout_container(): # run create-checkout result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) - # Check for successful command, creation of checkout script, and that script is executable (os.access - checks is file has specific access mode, os.X_OK - checks executable permission) + # Check for successful command, creation of checkout script, and that script is executable + # os.access - checks is file has specific access mode, os.X_OK - checks executable permission assert all ([result.exit_code == 0, Path(f"tmp/{platform}/checkout.sh").exists(), os.access(Path(f"tmp/{platform}/checkout.sh"), os.X_OK) == False ]) + +def test_cli_fre_make_create_checkout_cleanup(): + ''' make sure the checked out code doesnt stick around to mess up another pytest call ''' + shutil.rmtree(OUT_PATH) + assert not Path(OUT_PATH).exists() diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 7ad0c21c..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=64", "setuptools-scm>=8"] -build-backend = "setuptools.build_meta" From e5daf46b01e9f5a3a35392d5d2ca0d67854c83ee Mon Sep 17 00:00:00 2001 From: Ian <6273252+ilaflott@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:03:31 -0500 Subject: [PATCH 10/45] Update meta.yaml could it be? --- meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index f6956cf9..e58fd7bf 100644 --- a/meta.yaml +++ b/meta.yaml @@ -5,7 +5,8 @@ package: version: '{{ environ.get("GIT_DESCRIBE_TAG", data.get("version")) }}' source: - git_url: https://github.com/NOAA-GFDL/fre-cli.git + path: . +# git_url: https://github.com/NOAA-GFDL/fre-cli.git build: script: From fe0a656c6ff181b40149d3486928e4f1ccafa2b2 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 13:20:13 -0500 Subject: [PATCH 11/45] remove a sys.exit or two. restore os.environ[HOME] back to what it was pre- fre make cli tests. --- fre/pp/checkout_script.py | 13 +++++++++---- fre/pp/install_script.py | 4 ++-- fre/tests/test_fre_make_cli.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index 02dca1f7..a02cae14 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -11,7 +11,7 @@ import click -from fre import fre +from fre.fre import version as fre_ver FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' @@ -23,7 +23,7 @@ def checkout_template(experiment = None, platform = None, target = None, branch go_back_here = os.getcwd() # branch and version parameters - default_tag = fre.version + default_tag = fre_ver #fre.version git_clone_branch_arg = branch if branch is not None else default_tag if branch is None: print(f"(checkout_script) default tag is '{default_tag}'") @@ -32,6 +32,7 @@ def checkout_template(experiment = None, platform = None, target = None, branch # check args + set the name of the directory if None in [experiment, platform, target]: + os.chdir(go_back_here) raise ValueError( 'one of these are None: experiment / platform / target = \n' f'{experiment} / {platform} / {target}' ) name = f"{experiment}__{platform}__{target}" @@ -43,6 +44,8 @@ def checkout_template(experiment = None, platform = None, target = None, branch except Exception as exc: raise OSError( '(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc + finally: + os.chdir(go_back_here) checkout_exists = os.path.isdir(f'{directory}/{name}') @@ -72,13 +75,15 @@ def checkout_template(experiment = None, platform = None, target = None, branch else: print(f"(checkout_script) ERROR: checkout exists ('{directory}/{name}') and does not match '{git_clone_branch_arg}'") print(f"(checkout_script) ERROR: current branch is '{current_branch}', current tag-describe is '{current_tag}'") - exit(1) + #exit(1) + os.chdir(go_back_here) + raise ValueError('(checkout_script) neither tag nor branch matches the git clone branch arg') # make sure we are back where we should be if os.getcwd() != go_back_here: os.chdir(go_back_here) - return 0 + return ############################################# diff --git a/fre/pp/install_script.py b/fre/pp/install_script.py index 95f28eb5..64a245d6 100644 --- a/fre/pp/install_script.py +++ b/fre/pp/install_script.py @@ -27,9 +27,9 @@ def install_subtool(experiment, platform, target): if installed_def == source_def: print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") else: - print(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") print(f"ERROR: Please remove installed workflow with 'cylc clean {name}' or move the workflow run directory '{install_dir}'") - exit(1) + raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") + #exit(1) else: print(f"NOTE: About to install workflow into ~/cylc-run/{name}") cmd = f"cylc install --no-run-name {name}" diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 5e75307d..78e43241 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -8,6 +8,9 @@ runner = CliRunner() +def test_sanity1(): + assert Path(os.getcwd()).name == "fre-cli" + def test_cli_fre_make(): ''' fre make ''' result = runner.invoke(fre.fre, args=["make"]) @@ -40,11 +43,14 @@ def test_cli_fre_make_create_checkout_baremetal(): Path(OUT_PATH).mkdir(parents=True,exist_ok=True) # Set HOME for modelRoot location (output location) in fre make + old_home = os.environ["HOME"] os.environ["HOME"]=str(Path(OUT_PATH)) # run create-checkout result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + os.environ["HOME"] = old_home + # Check for successful command, creation of checkout script, and that script is executable # os.access - checks is file has specific access mode, os.X_OK - checks executable permission assert all ([result.exit_code == 0, @@ -58,16 +64,26 @@ def test_cli_fre_make_create_checkout_container(): platform = "hpcme.2023" target = "debug" + # Set HOME for modelRoot location (output location) in fre make + old_home = os.environ["HOME"] + os.environ["HOME"]=str(Path(OUT_PATH)) + # run create-checkout result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) + os.environ["HOME"] = old_home + # Check for successful command, creation of checkout script, and that script is executable # os.access - checks is file has specific access mode, os.X_OK - checks executable permission assert all ([result.exit_code == 0, Path(f"tmp/{platform}/checkout.sh").exists(), os.access(Path(f"tmp/{platform}/checkout.sh"), os.X_OK) == False ]) + +def test_sanity2(): + assert Path(os.getcwd()).name == "fre-cli" def test_cli_fre_make_create_checkout_cleanup(): ''' make sure the checked out code doesnt stick around to mess up another pytest call ''' + assert Path(OUT_PATH).exists() shutil.rmtree(OUT_PATH) assert not Path(OUT_PATH).exists() From 3260539a843be304b12a677526cc55746a40be56 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 13:43:05 -0500 Subject: [PATCH 12/45] oops --- fre/tests/test_fre_make_cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index 78e43241..a681964e 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -8,9 +8,6 @@ runner = CliRunner() -def test_sanity1(): - assert Path(os.getcwd()).name == "fre-cli" - def test_cli_fre_make(): ''' fre make ''' result = runner.invoke(fre.fre, args=["make"]) @@ -79,9 +76,6 @@ def test_cli_fre_make_create_checkout_container(): Path(f"tmp/{platform}/checkout.sh").exists(), os.access(Path(f"tmp/{platform}/checkout.sh"), os.X_OK) == False ]) -def test_sanity2(): - assert Path(os.getcwd()).name == "fre-cli" - def test_cli_fre_make_create_checkout_cleanup(): ''' make sure the checked out code doesnt stick around to mess up another pytest call ''' assert Path(OUT_PATH).exists() From 50498e26e85fac6b4b019620e2c956770a36213c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 15:21:16 -0500 Subject: [PATCH 13/45] update click scaffolding to be much more lean and mean. approach courtesy of Ray M., thank you ray. --- fre/cmor/cmor_lister.py | 15 --- fre/cmor/cmor_mixer.py | 14 +- fre/cmor/frecmor.py | 20 +-- run_test_file_cases.py | 274 +++++++++++++++++++--------------------- 4 files changed, 141 insertions(+), 182 deletions(-) diff --git a/fre/cmor/cmor_lister.py b/fre/cmor/cmor_lister.py index e7cfcd1d..1c71d42c 100644 --- a/fre/cmor/cmor_lister.py +++ b/fre/cmor/cmor_lister.py @@ -10,15 +10,10 @@ ''' -#import os import glob import json -#import shutil -#import subprocess from pathlib import Path -import click - DO_NOT_PRINT_LIST=[ 'comment', 'ok_min_mean_abs', 'ok_max_mean_abs', 'valid_min', 'valid_max' ] @@ -108,13 +103,3 @@ def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_v print(f'(FATAL) this line should be unreachable!!!') return - - -@click.command() -def _cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): - ''' entry point to fre cmor run for click. see cmor_list_subtool for argument descriptions.''' - return cmor_list_subtool(json_var_list, json_table_config_dir, opt_var_name) - - -if __name__ == '__main__': - cmor_list_subtool() diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index c561de71..d9efa582 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -14,7 +14,7 @@ import numpy as np import netCDF4 as nc -import click +#import click import cmor # ----- \start consts @@ -940,15 +940,3 @@ def cmor_run_subtool( indir = None, print(f'WARNING: DEBUG_MODE_RUN_ONE is True. breaking var_list loop') break return 0 - - -@click.command() -def _cmor_run_subtool(indir = None, - json_var_list = None, json_table_config = None, json_exp_config = None, - outdir = None, opt_var_name = None): - ''' entry point to fre cmor run for click. see cmor_run_subtool for argument descriptions.''' - return cmor_run_subtool(indir, json_var_list, json_table_config, json_exp_config, outdir, opt_var_name) - - -if __name__ == '__main__': - cmor_run_subtool() diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py index bc482ff3..8e42e2fb 100644 --- a/fre/cmor/frecmor.py +++ b/fre/cmor/frecmor.py @@ -2,8 +2,8 @@ import click -from .cmor_lister import _cmor_list_subtool -from .cmor_mixer import _cmor_run_subtool +from .cmor_lister import cmor_list_subtool +from .cmor_mixer import cmor_run_subtool OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \ "matching that variable name. I.e., this string help target local_vars, not " + \ @@ -47,14 +47,12 @@ def cmor_cli(): type = str, help=OPT_VAR_NAME_HELP, required=False) -@click.pass_context -def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name): +def run(indir, varlist, table_config, exp_config, outdir, opt_var_name): # pylint: disable=unused-argument """ Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing """ - context.invoke( - _cmor_run_subtool, + cmor_run_subtool( indir = indir, json_var_list = varlist, json_table_config = table_config, @@ -62,8 +60,6 @@ def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name) outdir = outdir, opt_var_name = opt_var_name ) - # context.forward( - # _cmor_run_subtool() ) @cmor_cli.command() @@ -79,8 +75,7 @@ def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name) type = str, help=OPT_VAR_NAME_HELP, required=False) -@click.pass_context -def list(context, varlist, table_config_dir, opt_var_name): +def list(varlist, table_config_dir, opt_var_name): ''' loop over json table files in config_dir and show which tables contain variables in var list/ the tool will also print what that table entry is expecting of that variable as well. if given @@ -98,13 +93,12 @@ def list(context, varlist, table_config_dir, opt_var_name): if opt_var_name is None and varlist is None: raise ValueError('opt_var_name and varlist cannot both be None') - context.invoke( - _cmor_list_subtool, + cmor_list_subtool( json_var_list = varlist, json_table_config_dir = table_config_dir, opt_var_name = opt_var_name ) - + if __name__ == "__main__": cmor_cli() diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 9adae56a..92ae40c5 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -26,7 +26,7 @@ def print_the_outcome(some_return,case_str): print('-----------------------------------------------------------------------------------------------------------------') print(f'\n\n\n\n\n\n\n\n\n\n') print_cwd() - #assert some_return == 0 + assert some_return == 0 # global consts for these tests, with no/trivial impact on the results ROOTDIR='fre/tests/test_files' @@ -72,142 +72,6 @@ def run_cmor_RUN(filename, table, opt_var_name): - - - - -## 1) SUCCEEDs -## land, Lmon, gr1 -#testfile_land_gr1_Lmon = \ -# '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/land/ts/monthly/5yr/' + \ -# 'land.005101-005512.lai.nc' -#try: -# some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -#except: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'land_gr1_Lmon / lai') -# -# -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') -# -# -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') -# -# -## 4) SUCCEEDs (no longitude coordinate case) -## atmos, AERmonZ / ta -## just like #1, but lack longitude -#testfile_atmos_gr1_AERmonZ_nolons = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ -# 'atmos_plev39_cmip.201001-201412.ta.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -# -# -## 5) SUCCEEDs -## ocean, Omon / sos -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') -# -# -## 7) SUCCEEDs -## ocean, Omon / so -#testfile_ocean_monthly_z_1x1deg_gr = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -#if some_return != 0: -# print('didnt pass ocean-file test number 7. exit.') -# sys.exit() -# -# -## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -## atmos, Amon / ch4global -#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_scalar/ts/monthly/5yr/' + \ -# 'atmos_scalar.197001-197412.ch4global.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -## pass -#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') -# -# -# -## 9) SUCCEEDs (needs coordinate variable axis with character string values) -## land, Emon / gppLut -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# pass -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -#if some_return != 0: -# print('didnt pass the land-file test. exit.') -# #sys.exit() - - - - - - #### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. #### for this case, that file is: #### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ @@ -226,8 +90,136 @@ def run_cmor_RUN(filename, table, opt_var_name): except Exception as exc: print(f'exception caught: exc=\n{exc}') some_return=-1 - pass + print_the_outcome(some_return,'ocean_monthly_gn / sos') -if some_return != 0: - print('didnt pass ocean-file test 6... exit.') -# sys.exit() + +sys.exit() + + + +# 1) SUCCEEDs +# land, Lmon, gr1 +testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' +try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') +except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'land_gr1_Lmon / lai') + + +# 2) SUCCEEDs +# atmos, Amon / cl +testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') + + +# 3) SUCCEEDs +# atmos, Amon / mc +testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') + + +# 4) SUCCEEDs (no longitude coordinate case) +# atmos, AERmonZ / ta +# just like #1, but lack longitude +testfile_atmos_gr1_AERmonZ_nolons = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') + + +# 5) SUCCEEDs +# ocean, Omon / sos +testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') + + +# 7) SUCCEEDs +# ocean, Omon / so +testfile_ocean_monthly_z_1x1deg_gr = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_z_1x1deg.000101-000512.so.nc' +try: + some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') + + +# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +# atmos, Amon / ch4global +testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' +try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + + + +# 9) SUCCEEDs (needs coordinate variable axis with character string values) +# land, Emon / gppLut +testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' +try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + +print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') + + + + + + From c92e4e849b1fbd350b69ab30a2f4c45656f64703 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 15:44:14 -0500 Subject: [PATCH 14/45] removing now-redundant click-scaffoldding from fre app, fre check, fre list, fre pp, fre run, and fre test --- fre/app/freapp.py | 27 ++++----- .../generate_time_averages.py | 2 - fre/app/mask_atmos_plevel.py | 3 +- fre/app/regrid_xy/regrid_xy.py | 24 +++----- fre/check/frecheck.py | 6 +- fre/check/frecheckexample.py | 3 - fre/list/frelist.py | 6 +- fre/list/frelistexample.py | 3 - fre/pp/checkout_script.py | 11 ---- fre/pp/configure_script_xml.py | 8 --- fre/pp/configure_script_yaml.py | 9 --- fre/pp/frepp.py | 57 +++++++------------ fre/pp/install_script.py | 5 -- fre/pp/run_script.py | 7 --- fre/pp/status_script.py | 7 --- fre/pp/trigger_script.py | 6 -- fre/pp/validate_script.py | 6 -- fre/pp/wrapper_script.py | 9 --- fre/run/frerun.py | 6 +- fre/run/frerunexample.py | 3 - fre/test/fretest.py | 6 +- fre/test/fretestexample.py | 3 - 22 files changed, 47 insertions(+), 170 deletions(-) diff --git a/fre/app/freapp.py b/fre/app/freapp.py index 29c34385..f946c86e 100644 --- a/fre/app/freapp.py +++ b/fre/app/freapp.py @@ -7,7 +7,7 @@ from .mask_atmos_plevel import mask_atmos_plevel_subtool from .generate_time_averages.generate_time_averages import generate -from .regrid_xy.regrid_xy import _regrid_xy +from .regrid_xy.regrid_xy import regrid_xy @click.group(help=click.style(" - access fre app subcommands", fg=(250,154,90))) def app_cli(): @@ -54,13 +54,11 @@ def app_cli(): help = "`defaultxyInterp` / `def_xy_interp` (env var) default lat/lon resolution " + \ "for output regridding. (change me? TODO)", required = True) -@click.pass_context -def regrid(context, - input_dir, output_dir, begin, tmp_dir, - remap_dir, source, grid_spec, def_xy_interp ): - # pylint: disable=unused-argument +def regrid( input_dir, output_dir, begin, tmp_dir, + remap_dir, source, grid_spec, def_xy_interp ): ''' regrid target netcdf file ''' - context.forward(_regrid_xy) + regrid_xy( input_dir, output_dir, begin, tmp_dir, + remap_dir, source, grid_spec, def_xy_interp ) @app_cli.command() @click.option("-i", "--infile", @@ -74,11 +72,9 @@ def regrid(context, @click.option("-p", "--psfile", # surface pressure... ps? TODO help = "Input NetCDF file containing surface pressure (ps)", required = True) -@click.pass_context -def mask_atmos_plevel(context, infile, outfile, psfile): - # pylint: disable = unused-argument +def mask_atmos_plevel(infile, outfile, psfile): """Mask out pressure level diagnostic output below land surface""" - context.forward(mask_atmos_plevel_subtool) + mask_atmos_plevel_subtool(infile, outfile, psfile) @app_cli.command() @@ -112,17 +108,14 @@ def mask_atmos_plevel(context, infile, outfile, psfile): type = click.Choice(["samp","pop","samp_mean","pop_mean"]), default = "samp", help = "Compute standard deviations for time-averages as well") -@click.pass_context -def gen_time_averages(context, inf, outf, pkg, var, unwgt, avg_type, stddev_type): - # pylint: disable = unused-argument +def gen_time_averages(inf, outf, pkg, var, unwgt, avg_type, stddev_type): """ generate time averages for specified set of netCDF files. Example: generate-time-averages.py /path/to/your/files/ """ start_time = time.perf_counter() - context.forward(generate) - # need to change to a click.echo, not sure if echo supports f strings - print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)') + generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type) + click.echo(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)') if __name__ == "__main__": app_cli() diff --git a/fre/app/generate_time_averages/generate_time_averages.py b/fre/app/generate_time_averages/generate_time_averages.py index 1c4bbf69..a5b75f05 100755 --- a/fre/app/generate_time_averages/generate_time_averages.py +++ b/fre/app/generate_time_averages/generate_time_averages.py @@ -1,5 +1,4 @@ ''' tools for generating time averages from various packages ''' -import click def generate_time_average(infile=None, outfile=None, pkg=None, var=None, @@ -42,7 +41,6 @@ def generate_time_average(infile=None, outfile=None, return exitstatus -@click.command() def generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type): ''' click entrypoint to time averaging routine ''' exitstatus=generate_time_average( inf, outf, diff --git a/fre/app/mask_atmos_plevel.py b/fre/app/mask_atmos_plevel.py index 0e7b2c28..f8dc2b30 100755 --- a/fre/app/mask_atmos_plevel.py +++ b/fre/app/mask_atmos_plevel.py @@ -10,9 +10,8 @@ import os import netCDF4 as nc import xarray as xr -import click -@click.command() + def mask_atmos_plevel_subtool(infile, outfile, psfile): ''' click entry point to fre cmor mask-atmos-plevel''' diff --git a/fre/app/regrid_xy/regrid_xy.py b/fre/app/regrid_xy/regrid_xy.py index 7b363e32..01881d59 100755 --- a/fre/app/regrid_xy/regrid_xy.py +++ b/fre/app/regrid_xy/regrid_xy.py @@ -13,7 +13,7 @@ #3rd party import metomi.rose.config as rose_cfg from netCDF4 import Dataset -import click + FREGRID_SHARED_FILES='/home/fms/shared_fregrid_remap_files' @@ -465,19 +465,9 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None, return 0 -@click.command() -def _regrid_xy( input_dir, output_dir, begin, tmp_dir, - remap_dir, source, grid_spec, def_xy_interp ): - """ click entrypoint """ - click.echo(f'(_regrid_xy) locals={locals()}') - click.echo( '(_regrid_xy) click entrypoint hit- calling regrid_xy()') - return regrid_xy( input_dir, output_dir, begin, tmp_dir, - remap_dir, source, grid_spec, def_xy_interp ) - - -def main(): - """steering, local test/debug""" - return regrid_xy() - -if __name__=='__main__': - main() +#def main(): +# """steering, local test/debug""" +# return regrid_xy() +# +#if __name__=='__main__': +# main() diff --git a/fre/check/frecheck.py b/fre/check/frecheck.py index 9640d138..d996a96b 100644 --- a/fre/check/frecheck.py +++ b/fre/check/frecheck.py @@ -10,11 +10,9 @@ def check_cli(): @check_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') -@click.pass_context -def function(context, uppercase): - # pylint: disable=unused-argument +def function(uppercase): """ - Execute fre check test """ - context.forward(check_test_function) + check_test_function(uppercase) if __name__ == "__main__": check_cli() diff --git a/fre/check/frecheckexample.py b/fre/check/frecheckexample.py index daac462a..8d4d7536 100644 --- a/fre/check/frecheckexample.py +++ b/fre/check/frecheckexample.py @@ -4,9 +4,6 @@ NOAA | GFDL """ -import click - -@click.command() def check_test_function(uppercase=None): """Execute fre list testfunction2.""" statement = "testingtestingtestingtesting" diff --git a/fre/list/frelist.py b/fre/list/frelist.py index fea3b6ea..01649a34 100644 --- a/fre/list/frelist.py +++ b/fre/list/frelist.py @@ -10,11 +10,9 @@ def list_cli(): @list_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') -@click.pass_context -def function(context, uppercase): - # pylint: disable=unused-argument +def function(uppercase): """ - Execute fre list test """ - context.forward(list_test_function) + list_test_function(uppercase) if __name__ == "__main__": list_cli() diff --git a/fre/list/frelistexample.py b/fre/list/frelistexample.py index 7e40e195..8423ed14 100644 --- a/fre/list/frelistexample.py +++ b/fre/list/frelistexample.py @@ -4,9 +4,6 @@ NOAA | GFDL """ -import click - -@click.command() def list_test_function(uppercase=None): """Execute fre list testfunction2.""" statement = "testingtestingtestingtesting" diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index a02cae14..295220f8 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -9,8 +9,6 @@ import sys import subprocess -import click - from fre.fre import version as fre_ver FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' @@ -87,14 +85,5 @@ def checkout_template(experiment = None, platform = None, target = None, branch ############################################# -@click.command() -def _checkout_template(experiment, platform, target, branch ): - ''' - Wrapper script for calling checkout_template - allows the decorated version - of the function to be separate from the undecorated version - ''' - return checkout_template(experiment, platform, target, branch) - - if __name__ == '__main__': checkout_template() diff --git a/fre/pp/configure_script_xml.py b/fre/pp/configure_script_xml.py index 4674a63b..1cf84340 100644 --- a/fre/pp/configure_script_xml.py +++ b/fre/pp/configure_script_xml.py @@ -14,7 +14,6 @@ import logging # third party -import click import metomi.rose.config import metomi.isodatetime.parsers @@ -684,12 +683,5 @@ def format_req_pp_year(pp_year): ############################################## -@click.command() -def _convert(): - ''' - Wrapper for convert call - allows users to call without command-line args - ''' - convert() - if __name__ == '__main__': convert() diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 25507c14..3ad68764 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -10,7 +10,6 @@ import os import json import shutil -import click from jsonschema import validate import yaml @@ -198,14 +197,6 @@ def yaml_info(yamlfile,experiment,platform,target): dumper(rose_remap, outfile) print(" " + outfile) -@click.command() -def _yaml_info(yamlfile,experiment,platform,target): - ''' - Wrapper script for calling yaml_info - allows the decorated version - of the function to be separate from the undecorated version - ''' - return yaml_info(yamlfile,experiment,platform,target) - # Use parseyaml function to parse created edits.yaml if __name__ == '__main__': yaml_info() diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index 55939232..c5f0bfbc 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -12,6 +12,8 @@ from fre.pp import status_script from fre.pp import wrapper_script + +# fre pp @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210))) def pp_cli(): ''' entry point to fre pp click commands ''' @@ -28,11 +30,9 @@ def pp_cli(): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.pass_context -def status(context, experiment, platform, target): - # pylint: disable=unused-argument +def status(experiment, platform, target): """ - Report status of PP configuration""" - context.forward(status_script._status_subtool) + status_script.status_subtool(experiment, platform, target) # fre pp run @pp_cli.command() @@ -45,11 +45,9 @@ def status(context, experiment, platform, target): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.pass_context -def run(context, experiment, platform, target): - # pylint: disable=unused-argument +def run(experiment, platform, target): """ - Run PP configuration""" - context.forward(run_script._pp_run_subtool) + run_script.pp_run_subtool(experiment, platform, target) # fre pp validate @pp_cli.command() @@ -62,11 +60,9 @@ def run(context, experiment, platform, target): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.pass_context -def validate(context, experiment, platform, target): - # pylint: disable=unused-argument +def validate(experiment, platform, target): """ - Validate PP configuration""" - context.forward(validate_script._validate_subtool) + validate_script.validate_subtool(experiment, platform, target): # fre pp install @pp_cli.command() @@ -79,11 +75,9 @@ def validate(context, experiment, platform, target): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.pass_context -def install(context, experiment, platform, target): - # pylint: disable=unused-argument +def install(experiment, platform, target): """ - Install PP configuration""" - context.forward(install_script._install_subtool) + install_script.install_subtool(experiment, platform, target) @pp_cli.command() @click.option("-y", "--yamlfile", type=str, @@ -98,11 +92,9 @@ def install(context, experiment, platform, target): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.pass_context -def configure_yaml(context,yamlfile,experiment,platform,target): - # pylint: disable=unused-argument +def configure_yaml(yamlfile,experiment,platform,target): """ - Execute fre pp configure """ - context.forward(configure_script_yaml._yaml_info) + configure_script_yaml.yaml_info(yamlfile,experiment,platform,target) @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -117,11 +109,9 @@ def configure_yaml(context,yamlfile,experiment,platform,target): @click.option("-b", "--branch", type =str, required=False, default = None, help="fre-workflows branch/tag to clone; default is $(fre --version)") -@click.pass_context -def checkout(context, experiment, platform, target, branch=None): - # pylint: disable=unused-argument +def checkout(experiment, platform, target, branch=None): """ - Execute fre pp checkout """ - context.forward(checkout_script._checkout_template) + checkout_script.checkout_template(experiment, platform, target, branch) @pp_cli.command() @click.option('-x', '--xml', @@ -174,12 +164,11 @@ def checkout(context, experiment, platform, target, branch=None): @click.option('--dual', is_flag=True, help="Optional. Append '_canopy' to pp, analysis, and refinediag dirs") -@click.pass_context -def configure_xml(context, xml, platform, target, experiment, do_analysis, historydir, refinedir, +def configure_xml(xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual): - # pylint: disable=unused-argument """ - Converts a Bronx XML to a Canopy rose-suite.conf """ - context.forward(configure_script_xml._convert) + configure_script_xml.convert(xml, platform, target, experiment, do_analysis, historydir, refinedir, + ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual) #fre pp wrapper @pp_cli.command() @@ -201,12 +190,10 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-t", "--time", required=False, default=None, help="Time whose history files are ready") -@click.pass_context -def wrapper(context, experiment, platform, target, config_file, branch, time): - # pylint: disable=unused-argument +def wrapper(experiment, platform, target, config_file, branch, time): """ - Execute fre pp steps in order """ print(f'(frepp.wrapper) about to foward context to wrapper.run_all_fre_pp_steps via click...') - context.forward(wrapper_script._run_all_fre_pp_steps) + wrapper_script.run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time) print(f'(frepp.wrapper) done fowarding context to wrapper.run_all_fre_pp_steps via click.') @pp_cli.command() @@ -222,11 +209,9 @@ def wrapper(context, experiment, platform, target, config_file, branch, time): @click.option("-t", "--time", required=True, help="Time whose history files are ready") -@click.pass_context -def trigger(context, experiment, platform, target, time): - # pylint: disable=unused-argument +def trigger(experiment, platform, target, time): """ - Start postprocessing for a particular time """ - context.forward(trigger_script._trigger) + trigger_script.trigger(experiment, platform, target, time) if __name__ == "__main__": ''' entry point for click to fre pp commands ''' diff --git a/fre/pp/install_script.py b/fre/pp/install_script.py index 64a245d6..2590a64c 100644 --- a/fre/pp/install_script.py +++ b/fre/pp/install_script.py @@ -3,7 +3,6 @@ from pathlib import Path import os import subprocess -import click def install_subtool(experiment, platform, target): """ @@ -35,7 +34,3 @@ def install_subtool(experiment, platform, target): cmd = f"cylc install --no-run-name {name}" subprocess.run(cmd, shell=True, check=True) -@click.command() -def _install_subtool(experiment, platform, target): - ''' entry point to install for click ''' - return install_subtool(experiment, platform, target) diff --git a/fre/pp/run_script.py b/fre/pp/run_script.py index b6ca607a..a5d7d54d 100644 --- a/fre/pp/run_script.py +++ b/fre/pp/run_script.py @@ -1,7 +1,6 @@ ''' fre pp run ''' import subprocess -import click def pp_run_subtool(experiment, platform, target): """ @@ -13,11 +12,5 @@ def pp_run_subtool(experiment, platform, target): cmd = f"cylc play {name}" subprocess.run(cmd, shell=True, check=True) -@click.command() -def _pp_run_subtool(experiment, platform, target): - ''' entry point to run for click ''' - return pp_run_subtool(experiment, platform, target) - - if __name__ == "__main__": pp_run_subtool() diff --git a/fre/pp/status_script.py b/fre/pp/status_script.py index b4f579c7..92d4ab3f 100644 --- a/fre/pp/status_script.py +++ b/fre/pp/status_script.py @@ -1,7 +1,6 @@ ''' fre pp status ''' import subprocess -import click TIMEOUT_SECS=120#30 @@ -15,11 +14,5 @@ def status_subtool(experiment, platform, target): cmd = f"cylc workflow-state {name}" subprocess.run(cmd, shell=True, check=True, timeout=TIMEOUT_SECS) - -@click.command() -def _status_subtool(experiment, platform, target): - ''' entry point to status for click ''' - return status_subtool(experiment, platform, target) - if __name__ == "__main__": status_subtool() diff --git a/fre/pp/trigger_script.py b/fre/pp/trigger_script.py index 14ce4291..f966c965 100644 --- a/fre/pp/trigger_script.py +++ b/fre/pp/trigger_script.py @@ -1,7 +1,6 @@ ''' fre pp trigger ''' import subprocess -import click def trigger(experiment, platform, target, time): """ @@ -13,10 +12,5 @@ def trigger(experiment, platform, target, time): subprocess.run(cmd, shell=True, check=True, timeout=30) -@click.command() -def _trigger(experiment, platform, target, time): - ''' entry point to trigger for click ''' - return trigger(experiment, platform, target, time) - if __name__ == "__main__": trigger() diff --git a/fre/pp/validate_script.py b/fre/pp/validate_script.py index d48f5f47..482466e8 100644 --- a/fre/pp/validate_script.py +++ b/fre/pp/validate_script.py @@ -2,7 +2,6 @@ import os import subprocess -import click def validate_subtool(experiment, platform, target): """ @@ -24,10 +23,5 @@ def validate_subtool(experiment, platform, target): subprocess.run(cmd, shell=True, check=True) os.chdir(go_back_here) -@click.command() -def _validate_subtool(experiment, platform, target): - ''' entry point to validate for click ''' - return validate_subtool(experiment, platform, target) - if __name__ == "__main__": validate_subtool() diff --git a/fre/pp/wrapper_script.py b/fre/pp/wrapper_script.py index 15a48dea..2cd3bd4d 100644 --- a/fre/pp/wrapper_script.py +++ b/fre/pp/wrapper_script.py @@ -11,8 +11,6 @@ # error handling import os -#import time -import click # Import from the local packages from fre.pp.checkout_script import checkout_template @@ -53,12 +51,5 @@ def run_all_fre_pp_steps(experiment, platform, target, config_file, branch=None, print('(run_all_fre_pp_steps) done.') -@click.command() -def _run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time): - ''' - click entry point for run_all_fre_pp_steps. - ''' - return run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time) - if __name__ == '__main__': run_all_fre_pp_steps() diff --git a/fre/run/frerun.py b/fre/run/frerun.py index c2c611c3..36dfac92 100644 --- a/fre/run/frerun.py +++ b/fre/run/frerun.py @@ -11,11 +11,9 @@ def run_cli(): @run_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') -@click.pass_context -def function(context, uppercase): - # pylint: disable=unused-argument +def function(uppercase): """ - Execute fre run test """ - context.forward(run_test_function) + run_test_function(uppercase) if __name__ == "__main__": run_cli() diff --git a/fre/run/frerunexample.py b/fre/run/frerunexample.py index a4335144..c5baa107 100644 --- a/fre/run/frerunexample.py +++ b/fre/run/frerunexample.py @@ -4,9 +4,6 @@ NOAA | GFDL """ -import click - -@click.command() def run_test_function(uppercase=None): """Execute fre run run_test_function""" statement = "testingtestingtestingtesting" diff --git a/fre/test/fretest.py b/fre/test/fretest.py index 172ac855..18015f73 100644 --- a/fre/test/fretest.py +++ b/fre/test/fretest.py @@ -11,11 +11,9 @@ def test_cli(): @test_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') -@click.pass_context -def function(context, uppercase): - # pylint: disable=unused-argument +def function(uppercase): """ - Execute fre test test """ - context.forward(test_test_function) + test_test_function(uppercase) if __name__ == "__main__": test_cli() diff --git a/fre/test/fretestexample.py b/fre/test/fretestexample.py index 86d0f6ce..4daa5447 100644 --- a/fre/test/fretestexample.py +++ b/fre/test/fretestexample.py @@ -4,9 +4,6 @@ NOAA | GFDL """ -import click - -@click.command() def test_test_function(uppercase=None): """Execute fre list test_test_function""" statement = "testingtestingtestingtesting" From 58c83c14aa4521f6d0b105155e95244b85c3eca3 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 15:57:49 -0500 Subject: [PATCH 15/45] fix an oops, apply same approach to fre/catalog --- fre/catalog/frecatalog.py | 21 +++++++++++---------- fre/pp/frepp.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/fre/catalog/frecatalog.py b/fre/catalog/frecatalog.py index 58149e5e..2a7141a0 100644 --- a/fre/catalog/frecatalog.py +++ b/fre/catalog/frecatalog.py @@ -30,33 +30,34 @@ def catalog_cli(): @click.option('--append', is_flag = True, default = False) @click.option('--slow', is_flag = True, default = False, help = "Open NetCDF files to retrieve additional vocabulary (standard_name and intrafile static variables") -@click.pass_context -def builder(context, input_path = None, output_path = None, config = None, filter_realm = None, +def builder(input_path = None, output_path = None, config = None, filter_realm = None, filter_freq = None, filter_chunk = None, overwrite = False, append = False, slow = False): # pylint: disable=unused-argument """ - Generate .csv and .json files for catalog """ - context.forward(gen_intake_gfdl.create_catalog_cli) - + gen_intake_gfdl.create_catalog_cli( + input_path, output_path, config, filter_realm, + filter_freq, filter_chunk, overwrite, append, slow) + @catalog_cli.command() @click.argument('json_path', nargs = 1 , required = True) @click.argument('json_template_path', nargs = 1 , required = False) @click.option('-tf', '--test-failure', is_flag=True, default = False, help="Errors are only printed. Program will not exit.") -@click.pass_context -def validate(context, json_path, json_template_path, test_failure): +def validate(json_path, json_template_path, test_failure): # pylint: disable=unused-argument """ - Validate a catalog against catalog schema """ - context.forward(test_catalog.main) + test_catalog.main(json_path, json_template_path, test_failure) + + @catalog_cli.command() @click.option('--input', required = True, multiple = True, help = 'Catalog json files to be merged, space-separated') @click.option('--output', required = True, nargs = 1, help = 'Merged catalog') -@click.pass_context -def merge(context, input, output): +def merge(input, output): """ - Merge two or more more catalogs into one """ - context.invoke(combine_cats.combine_cats, inputfiles=input, output_path=output) + combine_cats.combine_cats(inputfiles=input, output_path=output) if __name__ == "__main__": catalog_cli() diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index c5f0bfbc..ef9e6ff3 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -62,7 +62,7 @@ def run(experiment, platform, target): required=True) def validate(experiment, platform, target): """ - Validate PP configuration""" - validate_script.validate_subtool(experiment, platform, target): + validate_script.validate_subtool(experiment, platform, target) # fre pp install @pp_cli.command() From 3e0096e750ac0d4755109cc98f127600d2688aa9 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 16:13:48 -0500 Subject: [PATCH 16/45] apply click-redundancy-removal logic to fre make --- fre/make/create_checkout_script.py | 24 ++++++---------- fre/make/create_compile_script.py | 21 +++++--------- fre/make/create_docker_script.py | 21 +++++--------- fre/make/create_makefile_script.py | 28 +++++++------------ fre/make/fremake.py | 45 +++++++++++------------------- fre/make/run_fremake_script.py | 33 +++++++++------------- 6 files changed, 62 insertions(+), 110 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 336a6b85..7dd6a714 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -7,11 +7,11 @@ import subprocess import logging import sys -import click + import fre.yamltools.combine_yamls as cy from .gfdlfremake import varsfre, yamlfre, checkout, targetfre -def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, execute, verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -84,18 +84,18 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v if run: fre_checkout.run() else: - sys.exit() + return #sys.exit() else: - print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") + print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") # was click.echo if run: try: subprocess.run(args=[src_dir+"/checkout.sh"], check=True) except: - print("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.", - "\nTry removing test folder: " + platform["modelRoot"] +"\n") - raise + raise OSError("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.", + "\nTry removing test folder: " + platform["modelRoot"] +"\n") + else: - sys.exit() + return #sys.exit() else: src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" @@ -106,13 +106,5 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v fre_checkout.finish(pc) print("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n") -@click.command() -def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): - ''' - Decorator for calling checkout_create - allows the decorated version - of the function to be separate from the undecorated version - ''' - return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) - if __name__ == "__main__": checkout_create() diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index ed131bcc..22b407da 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,15 +1,16 @@ -#!/usr/bin/python3 - +''' +TODO: make docstring +''' import os import sys import logging from pathlib import Path from multiprocessing.dummy import Pool -import click + from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy -def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -79,21 +80,13 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() fremakeBuildList.append(fremakeBuild) - click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") + print("\nCompile script created at " + bldDir + "/compile.sh" + "\n") #was click.echo if run: if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool else: - sys.exit() - -@click.command() -def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): - ''' - Decorator for calling compile_create - allows the decorated version - of the function to be separate from the undecorated version - ''' - return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose) + return #sys.exit() if __name__ == "__main__": compile_create() diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index cde401c2..cce84616 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -1,15 +1,16 @@ -#!/usr/bin/python3 +''' +TODO- make docstring +''' import os import sys import subprocess from pathlib import Path -import click -#from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker + from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker import fre.yamltools.combine_yamls as cy -def dockerfile_create(yamlfile,platform,target,execute): +def dockerfile_create(yamlfile, platform, target, execute): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -67,24 +68,16 @@ def dockerfile_create(yamlfile,platform,target,execute): dockerBuild.writeRunscript(platform["RUNenv"],platform["containerRun"],tmpDir+"/execrunscript.sh") currDir = os.getcwd() - click.echo("\ntmpDir created in " + currDir + "/tmp") - click.echo("Dockerfile created in " + currDir +"\n") # create build script for container dockerBuild.createBuildScript(platform["containerBuild"], platform["containerRun"]) + print("\ntmpDir created in " + currDir + "/tmp") #was click.echo and a few lines above + print("Dockerfile created in " + currDir +"\n") #was click.echo and a few lines above print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") # run the script if option is given if run: subprocess.run(args=[dockerBuild.userScriptPath], check=True) -@click.command() -def _dockerfile_create(yamlfile,platform,target,execute): - ''' - Decorator for calling dockerfile_create - allows the decorated version - of the function to be separate from the undecorated version - ''' - return dockerfile_create(yamlfile,platform,target,execute) - if __name__ == "__main__": dockerfile_create() diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index 3a3ccaec..80b09287 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -1,14 +1,14 @@ -#!/usr/bin/python3 +''' +TODO doc-string +''' import os from pathlib import Path -import click - from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre import fre.yamltools.combine_yamls as cy -def makefile_create(yamlfile,platform,target): +def makefile_create(yamlfile, platform, target): """ Create the makefile """ @@ -25,13 +25,13 @@ def makefile_create(yamlfile,platform,target): ## If combined yaml exists, note message of its existence ## If combined yaml does not exist, combine model, compile, and platform yamls - full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) + full_combined = cy.combined_compile_existcheck(combined, yml, platform, target) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) ## Open the yaml file and parse as fremakeYaml - modelYaml = yamlfre.freyaml(full_combined,freVars) + modelYaml = yamlfre.freyaml(full_combined, freVars) fremakeYaml = modelYaml.getCompileYaml() fremakeBuildList = [] @@ -60,9 +60,9 @@ def makefile_create(yamlfile,platform,target): mkTemplatePath = platform["mkTemplate"]) # Loop through components and send the component name, requires, and overrides for the Makefile for c in fremakeYaml['src']: - freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + freMakefile.addComponent(c['component'], c['requires'], c['makeOverrides']) freMakefile.writeMakefile() - click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n") + print("\nMakefile created at " + bldDir + "/Makefile" + "\n") # was click.echo else: bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "./tmp/"+platformName @@ -75,17 +75,9 @@ def makefile_create(yamlfile,platform,target): # Loop through compenents and send the component name and requires for the Makefile for c in fremakeYaml['src']: - freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + freMakefile.addComponent(c['component'], c['requires'], c['makeOverrides']) freMakefile.writeMakefile() - click.echo("\nMakefile created at " + tmpDir + "/Makefile" + "\n") - -@click.command() -def _makefile_create(yamlfile,platform,target): - ''' - Decorator for calling makefile_create - allows the decorated version - of the function to be separate from the undecorated version - ''' - return makefile_create(yamlfile,platform,target) + print("\nMakefile created at " + tmpDir + "/Makefile" + "\n") # was click.echo if __name__ == "__main__": makefile_create() diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 5b013457..958b0a29 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -46,7 +46,7 @@ def make_cli(): type = str, help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", - multiple = True, # replaces nargs = -1, since click.option() + multiple = True, type = str, help = TARGET_OPT_HELP, required = True) @@ -75,13 +75,11 @@ def make_cli(): "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -@click.pass_context -def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose): - # pylint: disable=unused-argument +def run_fremake(yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose): """ - Perform all fremake functions to run checkout and compile model""" - context.forward(run_fremake_script._fremake_run) + run_fremake_script.fremake_run( + yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose) -#### @make_cli.command() @click.option("-y", "--yamlfile", @@ -117,15 +115,13 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -@click.pass_context -def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): - # pylint: disable=unused-argument +def create_checkout(yamlfile, platform, target, no_parallel_checkout, jobs, execute, verbose): """ - Write the checkout script """ - context.forward(create_checkout_script._checkout_create) + create_checkout_script.checkout_create( + yamlfile, platform, target, no_parallel_checkout, jobs, execute, verbose) -##### @make_cli.command -@click.option("-y", +@click.option("-y", "--yamlfile", type = str, help = YAMLFILE_OPT_HELP, @@ -140,13 +136,9 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e type = str, help = TARGET_OPT_HELP, required = True) -@click.pass_context -def create_makefile(context,yamlfile,platform,target): - # pylint: disable=unused-argument +def create_makefile(yamlfile, platform, target): """ - Write the makefile """ - context.forward(create_makefile_script._makefile_create) - -##### + create_makefile_script.makefile_create(yamlfile, platform, target) @make_cli.command @click.option("-y", @@ -183,36 +175,33 @@ def create_makefile(context,yamlfile,platform,target): "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -@click.pass_context -def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): - # pylint: disable=unused-argument +def create_compile(yamlfile, platform, target, jobs, parallel, execute, verbose) """ - Write the compile script """ - context.forward(create_compile_script._compile_create) + create_compile_script.compile_create( + yamlfile, platform, target, jobs, parallel, execute, verbose) @make_cli.command @click.option("-y", "--yamlfile", type = str, help = YAMLFILE_OPT_HELP, - required = True) # use click.option() over click.argument(), we want help statements + required = True) @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", - multiple = True, # replaces nargs = -1, since click.option() + multiple = True, type = str, help = TARGET_OPT_HELP, required = True) @click.option("--execute", is_flag = True, help = "Build Dockerfile that has been generated by create-docker.") -@click.pass_context -def create_dockerfile(context,yamlfile,platform,target,execute): - # pylint: disable=unused-argument +def create_dockerfile(yamlfile, platform, target, execute): """ - Write the dockerfile """ - context.forward(create_docker_script._dockerfile_create) + create_docker_script.dockerfile_create(yamlfile, platform, target, execute) if __name__ == "__main__": make_cli() diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index f1e44970..f0ab89c8 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -9,14 +9,14 @@ import logging from multiprocessing.dummy import Pool from pathlib import Path -import click + import subprocess import fre.yamltools.combine_yamls as cy from .gfdlfremake import ( targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal ) -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): +def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] @@ -48,13 +48,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec ## If combined yaml exists, note message of its existence ## If combined yaml does not exist, combine model, compile, and platform yamls - full_combined = cy.combined_compile_existcheck(combined,yml,platform,target) + full_combined = cy.combined_compile_existcheck(combined, yml, platform, target) ## Get the variables in the model yaml freVars = varsfre.frevars(full_combined) ## Open the yaml file and parse as fremakeYaml - modelYaml = yamlfre.freyaml(full_combined,freVars) + modelYaml = yamlfre.freyaml(full_combined, freVars) fremakeYaml = modelYaml.getCompileYaml() ## Error checking the targets @@ -80,8 +80,8 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec if not os.path.exists(srcDir): os.system("mkdir -p " + srcDir) if not os.path.exists(srcDir+"/checkout.sh"): - freCheckout = checkout.checkout("checkout.sh",srcDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout = checkout.checkout("checkout.sh", srcDir) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(), jobs, pc) freCheckout.finish(pc) os.chmod(srcDir+"/checkout.sh", 0o744) print("\nCheckout script created at "+ srcDir + "/checkout.sh \n") @@ -120,7 +120,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Loop through components, send component name/requires/overrides for Makefile for c in fremakeYaml['src']: - freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + freMakefile.addComponent(c['component'], c['requires'], c['makeOverrides']) print("\nMakefile created at " + bldDir + "/Makefile" + "\n") freMakefile.writeMakefile() @@ -152,7 +152,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec tmpDir = "tmp/"+platformName ## Create the checkout script freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) + freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(), jobs, pc) freCheckout.finish(pc) ## Create the makefile ### Should this even be a separate class from "makefile" in makefilefre? ~ ejs @@ -165,7 +165,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Loop through components and send the component name and requires for the Makefile for c in fremakeYaml['src']: - freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + freMakefile.addComponent(c['component'],c['requires'], c['makeOverrides']) freMakefile.writeMakefile() ## Build the dockerfile @@ -182,7 +182,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec for c in fremakeYaml['src']: dockerBuild.writeDockerfileMkmf(c) - dockerBuild.writeRunscript(platform["RUNenv"],platform["containerRun"],tmpDir+"/execrunscript.sh") + dockerBuild.writeRunscript(platform["RUNenv"], platform["containerRun"], tmpDir+"/execrunscript.sh") # Create build script for container dockerBuild.createBuildScript(platform["containerBuild"], platform["containerRun"]) @@ -193,7 +193,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec subprocess.run(args=[dockerBuild.userScriptPath], check=True) #freCheckout.cleanup() - #buildDockerfile(fremakeYaml,image) + #buildDockerfile(fremakeYaml, image) if baremetalRun: if __name__ == '__main__': @@ -201,15 +201,8 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Create a multiprocessing Pool pool = Pool(processes=nparallel) # process data_inputs iterable with pool - pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) - -@click.command() -def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): - ''' - Decorator for calling _fremake_run - allows the decorated version - of the function to be separate from the undecorated version - ''' - return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose) + pool.map(buildBaremetal.fremake_parallel, fremakeBuildList) + if __name__ == "__main__": fremake_run() From 19bc7a121e24645ee1a6af1abdd847be57fa60e9 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 16:26:30 -0500 Subject: [PATCH 17/45] Revert "fix an oops, apply same approach to fre/catalog" This reverts commit 58c83c14aa4521f6d0b105155e95244b85c3eca3. --- fre/catalog/frecatalog.py | 21 ++++++++++----------- fre/pp/frepp.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/fre/catalog/frecatalog.py b/fre/catalog/frecatalog.py index 2a7141a0..58149e5e 100644 --- a/fre/catalog/frecatalog.py +++ b/fre/catalog/frecatalog.py @@ -30,34 +30,33 @@ def catalog_cli(): @click.option('--append', is_flag = True, default = False) @click.option('--slow', is_flag = True, default = False, help = "Open NetCDF files to retrieve additional vocabulary (standard_name and intrafile static variables") -def builder(input_path = None, output_path = None, config = None, filter_realm = None, +@click.pass_context +def builder(context, input_path = None, output_path = None, config = None, filter_realm = None, filter_freq = None, filter_chunk = None, overwrite = False, append = False, slow = False): # pylint: disable=unused-argument """ - Generate .csv and .json files for catalog """ - gen_intake_gfdl.create_catalog_cli( - input_path, output_path, config, filter_realm, - filter_freq, filter_chunk, overwrite, append, slow) - + context.forward(gen_intake_gfdl.create_catalog_cli) + @catalog_cli.command() @click.argument('json_path', nargs = 1 , required = True) @click.argument('json_template_path', nargs = 1 , required = False) @click.option('-tf', '--test-failure', is_flag=True, default = False, help="Errors are only printed. Program will not exit.") -def validate(json_path, json_template_path, test_failure): +@click.pass_context +def validate(context, json_path, json_template_path, test_failure): # pylint: disable=unused-argument """ - Validate a catalog against catalog schema """ - test_catalog.main(json_path, json_template_path, test_failure) - - + context.forward(test_catalog.main) @catalog_cli.command() @click.option('--input', required = True, multiple = True, help = 'Catalog json files to be merged, space-separated') @click.option('--output', required = True, nargs = 1, help = 'Merged catalog') -def merge(input, output): +@click.pass_context +def merge(context, input, output): """ - Merge two or more more catalogs into one """ - combine_cats.combine_cats(inputfiles=input, output_path=output) + context.invoke(combine_cats.combine_cats, inputfiles=input, output_path=output) if __name__ == "__main__": catalog_cli() diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index ef9e6ff3..c5f0bfbc 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -62,7 +62,7 @@ def run(experiment, platform, target): required=True) def validate(experiment, platform, target): """ - Validate PP configuration""" - validate_script.validate_subtool(experiment, platform, target) + validate_script.validate_subtool(experiment, platform, target): # fre pp install @pp_cli.command() From aef76eeb45dbf8137b143d28e9246b9262828190 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 16:27:43 -0500 Subject: [PATCH 18/45] remove needless shebang, put fre catalog click scaffolding back where it was. catalogbuilder may need context forwarding. --- fre/catalog/tests/test_fre_catalog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fre/catalog/tests/test_fre_catalog.py b/fre/catalog/tests/test_fre_catalog.py index 7bc36637..ea400601 100644 --- a/fre/catalog/tests/test_fre_catalog.py +++ b/fre/catalog/tests/test_fre_catalog.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 def test_fre_catalog_import(): from fre import catalog From e03e0986e400b7848a6b19e5646b8813c9bb73da Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 16:32:43 -0500 Subject: [PATCH 19/45] fix the silly colon-oops in fre.pp again --- fre/pp/frepp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index c5f0bfbc..ef9e6ff3 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -62,7 +62,7 @@ def run(experiment, platform, target): required=True) def validate(experiment, platform, target): """ - Validate PP configuration""" - validate_script.validate_subtool(experiment, platform, target): + validate_script.validate_subtool(experiment, platform, target) # fre pp install @pp_cli.command() From e832807e8030e7dfd3fea5259dece12655ba5965 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 16:44:35 -0500 Subject: [PATCH 20/45] fix another syntax oops. change sys.exit to return --- fre/make/create_checkout_script.py | 6 +++--- fre/make/create_compile_script.py | 2 +- fre/make/fremake.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 7dd6a714..3fafacf8 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -84,9 +84,9 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec if run: fre_checkout.run() else: - return #sys.exit() + return #0 #sys.exit() else: - print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") # was click.echo + print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") if run: try: subprocess.run(args=[src_dir+"/checkout.sh"], check=True) @@ -95,7 +95,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec "\nTry removing test folder: " + platform["modelRoot"] +"\n") else: - return #sys.exit() + return #0 #sys.exit() else: src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src" diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index 22b407da..6dbebd44 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -86,7 +86,7 @@ def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose) pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool else: - return #sys.exit() + return #0 #sys.exit() if __name__ == "__main__": compile_create() diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 958b0a29..0593445c 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -175,7 +175,7 @@ def create_makefile(yamlfile, platform, target): "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -def create_compile(yamlfile, platform, target, jobs, parallel, execute, verbose) +def create_compile(yamlfile, platform, target, jobs, parallel, execute, verbose): """ - Write the compile script """ create_compile_script.compile_create( yamlfile, platform, target, jobs, parallel, execute, verbose) From 1a01de957c9c9a54752ccc91a2dcc22b745c16b4 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 17:14:43 -0500 Subject: [PATCH 21/45] more getting tests etc to clean up after themselves --- fre/app/regrid_xy/regrid_xy.py | 12 ++++---- fre/cmor/cmor_mixer.py | 3 +- fre/make/create_checkout_script.py | 2 +- fre/pp/tests/test_configure_script_yaml.py | 33 ++++++++++++++-------- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/fre/app/regrid_xy/regrid_xy.py b/fre/app/regrid_xy/regrid_xy.py index 01881d59..9ab5973f 100755 --- a/fre/app/regrid_xy/regrid_xy.py +++ b/fre/app/regrid_xy/regrid_xy.py @@ -465,9 +465,9 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None, return 0 -#def main(): -# """steering, local test/debug""" -# return regrid_xy() -# -#if __name__=='__main__': -# main() +def main(): + """steering, local test/debug""" + return regrid_xy() + +if __name__=='__main__': + main() diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index d9efa582..53bd3989 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -14,7 +14,6 @@ import numpy as np import netCDF4 as nc -#import click import cmor # ----- \start consts @@ -32,7 +31,7 @@ def from_dis_gimme_dis(from_dis, gimme_dis): return from_dis[gimme_dis][:].copy() except Exception as exc: print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' -# f' from this: {from_dis} ' + f' from this: {from_dis} ' f' exc = {exc}' f' returning None!' ) return None diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 3fafacf8..8f5ef379 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -84,7 +84,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec if run: fre_checkout.run() else: - return #0 #sys.exit() + return else: print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") if run: diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index e6391513..a825275b 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -2,6 +2,7 @@ Test configure_script_yaml """ import os +import shutil from pathlib import Path from fre.pp import configure_script_yaml as csy @@ -11,14 +12,14 @@ TARGET = "prod-openmp" # Set example yaml paths, input directory -test_dir = Path("fre/pp/tests") -test_yaml = Path("AM5_example/am5.yaml") +TEST_DIR = Path("fre/pp/tests") +TEST_YAML = Path("AM5_example/am5.yaml") def test_combinedyaml_exists(): """ Make sure combined yaml file exists """ - assert Path(f"{test_dir}/{test_yaml}").exists() + assert Path(f"{TEST_DIR}/{TEST_YAML}").exists() def test_configure_script(): """ @@ -27,20 +28,28 @@ def test_configure_script(): TO-DO: will break this up for better tests """ # Set home for ~/cylc-src location in script - os.environ["HOME"]=str(Path(f"{test_dir}/configure_yaml_out")) + old_home = os.environ["HOME"] + os.environ["HOME"] = str(Path(f"{TEST_DIR}/configure_yaml_out")) # Set output directory - out_dir = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") - Path(out_dir).mkdir(parents=True,exist_ok=True) + OUT_DIR = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}") + Path(OUT_DIR).mkdir(parents = True, exist_ok = True) # Define combined yaml - model_yaml = str(Path(f"{test_dir}/{test_yaml}")) + model_yaml = str(Path(f"{TEST_DIR}/{TEST_YAML}")) # Invoke configure_yaml_script.py - csy.yaml_info(model_yaml,EXPERIMENT,PLATFORM,TARGET) + csy.yaml_info(model_yaml, EXPERIMENT, PLATFORM, TARGET) + + os.environ["HOME"] = old_home # Check for configuration creation and final combined yaml - assert all([Path(f"{out_dir}/{EXPERIMENT}.yaml").exists(), - Path(f"{out_dir}/rose-suite.conf").exists(), - Path(f"{out_dir}/app/regrid-xy/rose-app.conf").exists(), - Path(f"{out_dir}/app/remap-pp-components/rose-app.conf").exists()]) + assert all([ Path(f"{OUT_DIR}/{EXPERIMENT}.yaml").exists(), + Path(f"{OUT_DIR}/rose-suite.conf").exists(), + Path(f"{OUT_DIR}/app/regrid-xy/rose-app.conf").exists(), + Path(f"{OUT_DIR}/app/remap-pp-components/rose-app.conf").exists() ]) + +def test_cleanup(): + shutil.rmtree(f"{TEST_DIR}/configure_yaml_out") + assert not Path(f"{TEST_DIR}/configure_yaml_out").exists() + From ada0dc7eaf3e0ebe81dd3c15657bb94208fb2bd5 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 17:25:22 -0500 Subject: [PATCH 22/45] #no more shebangs! (getit?) --- fre/analysis/freanalysis.py | 10 +++++++++- fre/app/freapp.py | 1 - fre/app/mask_atmos_plevel.py | 1 - fre/catalog/frecatalog.py | 1 + fre/fre.py | 17 ++++++++--------- fre/make/gfdlfremake/buildBaremetal.py | 11 ++++++----- fre/make/gfdlfremake/buildDocker.py | 11 ++++++----- fre/pp/configure_script_yaml.py | 3 --- 8 files changed, 30 insertions(+), 25 deletions(-) diff --git a/fre/analysis/freanalysis.py b/fre/analysis/freanalysis.py index 212b13d5..c135ee91 100644 --- a/fre/analysis/freanalysis.py +++ b/fre/analysis/freanalysis.py @@ -1,6 +1,14 @@ -from analysis_scripts import available_plugins +''' +TODO: add doc-string +''' + +# a third party package import click +# a diff gfdl package +from analysis_scripts import available_plugins + +# this package from .subtools import install_analysis_package, list_plugins, run_analysis, \ uninstall_analysis_package diff --git a/fre/app/freapp.py b/fre/app/freapp.py index f946c86e..6564f91d 100644 --- a/fre/app/freapp.py +++ b/fre/app/freapp.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 ''' fre app calls ''' import time diff --git a/fre/app/mask_atmos_plevel.py b/fre/app/mask_atmos_plevel.py index f8dc2b30..5ec9c48b 100755 --- a/fre/app/mask_atmos_plevel.py +++ b/fre/app/mask_atmos_plevel.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' This script contains the refineDiags that produce data at the same frequency as the input data (no reduction) such as surface albedo, diff --git a/fre/catalog/frecatalog.py b/fre/catalog/frecatalog.py index 58149e5e..51340ca4 100644 --- a/fre/catalog/frecatalog.py +++ b/fre/catalog/frecatalog.py @@ -3,6 +3,7 @@ ''' import click + #import catalogbuilder from catalogbuilder.scripts import gen_intake_gfdl from catalogbuilder.scripts import test_catalog diff --git a/fre/fre.py b/fre/fre.py index 61ebb73c..249a06c7 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -7,9 +7,14 @@ be called via this script. I.e. 'fre' is the entry point """ -#versioning... always fun... -# turn xxxx.y into xxxx.0y import importlib.metadata + +import click +from .lazy_group import LazyGroup + + + +# versioning, turn xxxx.y into xxxx.0y version_unexpanded = importlib.metadata.version('fre-cli') version_unexpanded_split = version_unexpanded.split('.') if len(version_unexpanded_split[1]) == 1: @@ -18,13 +23,7 @@ version_minor = version_unexpanded_split[1] version = version_unexpanded_split[0] + '.' + version_minor - - - - -import click -from .lazy_group import LazyGroup - +# click and lazy group loading @click.group( cls = LazyGroup, lazy_subcommands = {"pp": ".pp.frepp.pp_cli", diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index 9e742980..3159baf6 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -1,8 +1,9 @@ -#!/usr/bin/python3 -## \date 2023 -## \author Tom Robinson -## \email thomas.robinson@noaa.gov -## \description +''' + \date 2023 + \author Tom Robinson + \email thomas.robinson@noaa.gov + \description +''' import subprocess import os diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index 491082ef..bf48b1f9 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -1,8 +1,9 @@ -#!/usr/bin/python3 -## \date 2023 -## \author Tom Robinson -## \email thomas.robinson@noaa.gov -## \description +''' + \date 2023 + \author Tom Robinson + \email thomas.robinson@noaa.gov + \description +''' import os diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 3ad68764..24d5be00 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -15,9 +15,6 @@ import yaml import metomi.rose.config -# Relative import -#f = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -#sys.path.append(f) import fre.yamltools.combine_yamls as cy #################### From ab2708e46140c7493baa8f87edd4c440d9796a35 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 17:41:09 -0500 Subject: [PATCH 23/45] edits addressing pylint complains about no value for arguments in calls --- fre/make/tests/compilation/test_run_fremake_builds.py | 6 +++--- fre/pp/run_script.py | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 1be4731c..3967323a 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -1,6 +1,6 @@ -''' this file holds any run-fremake tests that actually compile the model code''' -''' these tests assume your os is the ci image (gcc 14 + mpich on rocky 8)''' -''' you may need to add mkmf to your path or make other adjustments to the mkmf template to run elsewhere''' +''' this file holds any run-fremake tests that actually compile the model code + these tests assume your os is the ci image (gcc 14 + mpich on rocky 8) + you may need to add mkmf to your path or make other adjustments to the mkmf template to run elsewhere''' import os from shutil import rmtree diff --git a/fre/pp/run_script.py b/fre/pp/run_script.py index a5d7d54d..0865ff10 100644 --- a/fre/pp/run_script.py +++ b/fre/pp/run_script.py @@ -2,11 +2,15 @@ import subprocess -def pp_run_subtool(experiment, platform, target): +def pp_run_subtool(experiment = None, platform = None, target = None): """ Start or restart the Cylc workflow identified by: ____ """ + if None in [experiment, platform, target]: + raise ValueError( 'experiment, platform, and target must all not be None.' + 'currently, their values are...' + f'{experiment} / {platform} / {target}') name = experiment + '__' + platform + '__' + target cmd = f"cylc play {name}" From bc4a39b8d434b78ab717d470495653c1bb913af3 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 17:48:45 -0500 Subject: [PATCH 24/45] further edits addressing pylint complains about no value for arguments in calls --- fre/pp/configure_script_yaml.py | 6 +++++- fre/pp/status_script.py | 6 +++++- fre/pp/trigger_script.py | 4 ++++ fre/pp/validate_script.py | 7 ++++++- fre/pp/wrapper_script.py | 2 +- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 24d5be00..a690a701 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -142,13 +142,17 @@ def set_rose_apps(yamlfile,rose_regrid,rose_remap): value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') #################### -def yaml_info(yamlfile,experiment,platform,target): +def yaml_info(yamlfile = None, experiment = None, platform = None, target = None): """ Using a valid pp.yaml, the rose-app and rose-suite configuration files are created in the cylc-src directory. The pp.yaml is also copied to the cylc-src directory. """ + if None in [yamlfile, experiment, platform, target]: + raise ValueError( 'yamlfile, experiment, platform, and target must all not be None.' + 'currently, their values are...' + f'{yamlfile} / {experiment} / {platform} / {target}') e = experiment p = platform t = target diff --git a/fre/pp/status_script.py b/fre/pp/status_script.py index 92d4ab3f..273459b4 100644 --- a/fre/pp/status_script.py +++ b/fre/pp/status_script.py @@ -4,11 +4,15 @@ TIMEOUT_SECS=120#30 -def status_subtool(experiment, platform, target): +def status_subtool(experiment = None, platform = None, target = None): """ Report workflow state for the Cylc workflow ____ """ + if None in [experiment, platform, target]: + raise ValueError( 'experiment, platform, and target must all not be None.' + 'currently, their values are...' + f'{experiment} / {platform} / {target}') name = experiment + '__' + platform + '__' + target cmd = f"cylc workflow-state {name}" diff --git a/fre/pp/trigger_script.py b/fre/pp/trigger_script.py index f966c965..11752456 100644 --- a/fre/pp/trigger_script.py +++ b/fre/pp/trigger_script.py @@ -6,6 +6,10 @@ def trigger(experiment, platform, target, time): """ Trigger the pp-starter task for the time indicated """ + if None in [experiment, platform, target, time]: + raise ValueError( 'experiment, platform, target and time must all not be None.' + 'currently, their values are...' + f'{experiment} / {platform} / {target} / {time}') name = experiment + '__' + platform + '__' + target cmd = f"cylc trigger {name}//{time}/pp-starter" diff --git a/fre/pp/validate_script.py b/fre/pp/validate_script.py index 482466e8..dd7e5dca 100644 --- a/fre/pp/validate_script.py +++ b/fre/pp/validate_script.py @@ -3,11 +3,16 @@ import os import subprocess -def validate_subtool(experiment, platform, target): +def validate_subtool(experiment = None, platform = None, target = None): """ Validate the Cylc workflow definition located in ~/cylc-src/____ """ + if None in [experiment, platform, target]: + raise ValueError( 'experiment, platform, and target must all not be None.' + 'currently, their values are...' + f'{experiment} / {platform} / {target}') + go_back_here = os.getcwd() directory = os.path.expanduser('~/cylc-src/' + experiment + '__' + platform + '__' + target) diff --git a/fre/pp/wrapper_script.py b/fre/pp/wrapper_script.py index 2cd3bd4d..a077e823 100644 --- a/fre/pp/wrapper_script.py +++ b/fre/pp/wrapper_script.py @@ -20,7 +20,7 @@ from fre.pp.trigger_script import trigger from fre.pp.status_script import status_subtool -def run_all_fre_pp_steps(experiment, platform, target, config_file, branch=None, time=None): +def run_all_fre_pp_steps(experiment = None, platform = None, target = None, config_file, branch = None, time = None): ''' Wrapper script for calling a FRE2 pp experiment with the canopy-style infrastructure and fre-cli From 67b67ee7c454e8521106befdacc67a1d9d81e382 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 18:02:18 -0500 Subject: [PATCH 25/45] notimplementederrors raised on functions not-yet-used. uncomment the help function test in meta.yaml because we DO want those to work --- fre/check/frecheck.py | 3 ++- fre/list/frelist.py | 3 ++- fre/pp/wrapper_script.py | 2 +- fre/run/frerun.py | 3 ++- fre/test/fretest.py | 1 + meta.yaml | 16 ++++++++-------- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/fre/check/frecheck.py b/fre/check/frecheck.py index d996a96b..62e944b3 100644 --- a/fre/check/frecheck.py +++ b/fre/check/frecheck.py @@ -4,7 +4,7 @@ from .frecheckexample import check_test_function -@click.group(help=click.style(" - access fre check subcommands", fg=(162,91,232))) +@click.group(help=click.style(" - access fre check subcommands (not implemented yet!)", fg=(162,91,232))) def check_cli(): ''' entry point to fre check click commands ''' @@ -13,6 +13,7 @@ def check_cli(): def function(uppercase): """ - Execute fre check test """ check_test_function(uppercase) + raise NotImplementedError('fre check has not been implemented yet!') if __name__ == "__main__": check_cli() diff --git a/fre/list/frelist.py b/fre/list/frelist.py index 01649a34..91053df0 100644 --- a/fre/list/frelist.py +++ b/fre/list/frelist.py @@ -4,7 +4,7 @@ from .frelistexample import list_test_function -@click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91))) +@click.group(help=click.style(" - access fre list subcommands (not implemented yet!)", fg=(232,204,91))) def list_cli(): ''' entry point to fre list click commands ''' @@ -13,6 +13,7 @@ def list_cli(): def function(uppercase): """ - Execute fre list test """ list_test_function(uppercase) + raise NotImplementedError('fre list has not been implemented yet!') if __name__ == "__main__": list_cli() diff --git a/fre/pp/wrapper_script.py b/fre/pp/wrapper_script.py index a077e823..b79d847d 100644 --- a/fre/pp/wrapper_script.py +++ b/fre/pp/wrapper_script.py @@ -20,7 +20,7 @@ from fre.pp.trigger_script import trigger from fre.pp.status_script import status_subtool -def run_all_fre_pp_steps(experiment = None, platform = None, target = None, config_file, branch = None, time = None): +def run_all_fre_pp_steps(experiment = None, platform = None, target = None, config_file = None, branch = None, time = None): ''' Wrapper script for calling a FRE2 pp experiment with the canopy-style infrastructure and fre-cli diff --git a/fre/run/frerun.py b/fre/run/frerun.py index 36dfac92..f39c2a24 100644 --- a/fre/run/frerun.py +++ b/fre/run/frerun.py @@ -5,7 +5,7 @@ import click from .frerunexample import run_test_function -@click.group(help=click.style(" - access fre run subcommands", fg=(164,29,132))) +@click.group(help=click.style(" - access fre run subcommands (not implemented yet!)", fg=(164,29,132))) def run_cli(): ''' entry point to fre run click commands ''' @@ -14,6 +14,7 @@ def run_cli(): def function(uppercase): """ - Execute fre run test """ run_test_function(uppercase) + raise NotImplementedError('fre run has not been implemented yet!') if __name__ == "__main__": run_cli() diff --git a/fre/test/fretest.py b/fre/test/fretest.py index 18015f73..e77aad06 100644 --- a/fre/test/fretest.py +++ b/fre/test/fretest.py @@ -14,6 +14,7 @@ def test_cli(): def function(uppercase): """ - Execute fre test test """ test_test_function(uppercase) + raise NotImplementedError('fre test has not been implemented yet!') if __name__ == "__main__": test_cli() diff --git a/meta.yaml b/meta.yaml index c782da9c..2c9fdcde 100644 --- a/meta.yaml +++ b/meta.yaml @@ -56,13 +56,13 @@ test: - fre - fre.app - fre.catalog - # - fre.check + - fre.check - fre.cmor - # - fre.list + - fre.list - fre.make - fre.pp - # - fre.run - # - fre.test + - fre.run + - fre.test - fre.yamltools commands: - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." @@ -71,13 +71,13 @@ test: - fre --help - fre app --help - fre catalog --help - # - fre check --help + - fre check --help - fre cmor --help - # - fre list --help + - fre list --help - fre make --help - fre pp --help - # - fre run --help - # - fre test --help + - fre run --help + - fre test --help - fre yamltools --help From a074922df18013617509763912a4041729e0da4b Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 18:15:35 -0500 Subject: [PATCH 26/45] remove useless __init__ files in test directories. more pylint edits. --- fre/analysis/freanalysis.py | 10 +++------- fre/analysis/tests/__init__.py | 0 fre/app/generate_time_averages/tests/__init__.py | 0 fre/catalog/tests/__init__.py | 0 fre/cmor/tests/__init__.py | 0 fre/make/tests/__init__.py | 0 fre/pp/checkout_script.py | 5 ++--- fre/pp/install_script.py | 3 +-- fre/pp/tests/__init__.py | 0 fre/pp/trigger_script.py | 2 +- fre/yamltools/tests/__init__.py | 0 11 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 fre/analysis/tests/__init__.py delete mode 100644 fre/app/generate_time_averages/tests/__init__.py delete mode 100644 fre/catalog/tests/__init__.py delete mode 100644 fre/cmor/tests/__init__.py delete mode 100644 fre/make/tests/__init__.py delete mode 100644 fre/pp/tests/__init__.py delete mode 100644 fre/yamltools/tests/__init__.py diff --git a/fre/analysis/freanalysis.py b/fre/analysis/freanalysis.py index c135ee91..3a85ea06 100644 --- a/fre/analysis/freanalysis.py +++ b/fre/analysis/freanalysis.py @@ -1,12 +1,8 @@ -''' -TODO: add doc-string -''' - # a third party package import click -# a diff gfdl package -from analysis_scripts import available_plugins +## a diff gfdl package +#from analysis_scripts import available_plugins # this package from .subtools import install_analysis_package, list_plugins, run_analysis, \ @@ -16,7 +12,7 @@ @click.group(help=click.style(" - access fre analysis subcommands", fg=(250, 154, 90))) def analysis_cli(): """Entry point to fre analysis click commands.""" - pass + @analysis_cli.command() diff --git a/fre/analysis/tests/__init__.py b/fre/analysis/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/app/generate_time_averages/tests/__init__.py b/fre/app/generate_time_averages/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/catalog/tests/__init__.py b/fre/catalog/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/cmor/tests/__init__.py b/fre/cmor/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/make/tests/__init__.py b/fre/make/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index 295220f8..c374b6c2 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -73,9 +73,8 @@ def checkout_template(experiment = None, platform = None, target = None, branch else: print(f"(checkout_script) ERROR: checkout exists ('{directory}/{name}') and does not match '{git_clone_branch_arg}'") print(f"(checkout_script) ERROR: current branch is '{current_branch}', current tag-describe is '{current_tag}'") - #exit(1) - os.chdir(go_back_here) - raise ValueError('(checkout_script) neither tag nor branch matches the git clone branch arg') + os.chdir(go_back_here) + raise ValueError('(checkout_script) neither tag nor branch matches the git clone branch arg') #exit(1) # make sure we are back where we should be if os.getcwd() != go_back_here: diff --git a/fre/pp/install_script.py b/fre/pp/install_script.py index 2590a64c..008b2722 100644 --- a/fre/pp/install_script.py +++ b/fre/pp/install_script.py @@ -27,8 +27,7 @@ def install_subtool(experiment, platform, target): print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") else: print(f"ERROR: Please remove installed workflow with 'cylc clean {name}' or move the workflow run directory '{install_dir}'") - raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") - #exit(1) + raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") #exit(1) else: print(f"NOTE: About to install workflow into ~/cylc-run/{name}") cmd = f"cylc install --no-run-name {name}" diff --git a/fre/pp/tests/__init__.py b/fre/pp/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fre/pp/trigger_script.py b/fre/pp/trigger_script.py index 11752456..b5cbdc35 100644 --- a/fre/pp/trigger_script.py +++ b/fre/pp/trigger_script.py @@ -2,7 +2,7 @@ import subprocess -def trigger(experiment, platform, target, time): +def trigger(experiment = None, platform = None, target = None, time = None): """ Trigger the pp-starter task for the time indicated """ diff --git a/fre/yamltools/tests/__init__.py b/fre/yamltools/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 From fb6249885c56d905caa68e3a1179d0ced50faf79 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 6 Jan 2025 19:34:40 -0500 Subject: [PATCH 27/45] why last failure? unconvinced. fail again, conda_build, i dare you. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e195bdc..a47303a6 100644 --- a/README.md +++ b/README.md @@ -87,4 +87,4 @@ To be developed: - [x] **fre pp** - [ ] **fre run** - [ ] **fre test** -- [ ] **fre yamltools** +- [x] **fre yamltools** \ No newline at end of file From 21e22b45c746aa0d6dbd18f9b452482a33a83a6b Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 7 Jan 2025 12:15:28 -0500 Subject: [PATCH 28/45] adjust import statement in fre/cmor/tests --- fre/cmor/tests/test_cmor_run_subtool.py | 16 +++++--- run_test_file_cases.py | 54 ++++++++++++------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/fre/cmor/tests/test_cmor_run_subtool.py b/fre/cmor/tests/test_cmor_run_subtool.py index 11ecb018..a896f250 100644 --- a/fre/cmor/tests/test_cmor_run_subtool.py +++ b/fre/cmor/tests/test_cmor_run_subtool.py @@ -4,7 +4,9 @@ from pathlib import Path from datetime import date -import fre +#import fre +#from fre import cmor +from fre.cmor import cmor_run_subtool import subprocess @@ -74,9 +76,13 @@ def test_setup_fre_cmor_run_subtool(capfd): def test_fre_cmor_run_subtool_case1(capfd): ''' fre cmor run, test-use case ''' + #import sys + #assert False, f'{sys.path}' + + #debug #print( - # f"fre.cmor.cmor_run_subtool(" + # f"cmor_run_subtool(" # f"\'{INDIR}\'," # f"\'{VARLIST}\'," # f"\'{TABLE_CONFIG}\'," @@ -86,7 +92,7 @@ def test_fre_cmor_run_subtool_case1(capfd): #) # test call, where meat of the workload gets done - fre.cmor.cmor_run_subtool( + cmor_run_subtool( indir = INDIR, json_var_list = VARLIST, json_table_config = TABLE_CONFIG, @@ -196,7 +202,7 @@ def test_fre_cmor_run_subtool_case2(capfd): #debug #print( - # f"fre.cmor.cmor_run_subtool(" + # f"cmor_run_subtool(" # f"\'{INDIR}\'," # f"\'{VARLIST_DIFF}\'," # f"\'{TABLE_CONFIG}\'," @@ -206,7 +212,7 @@ def test_fre_cmor_run_subtool_case2(capfd): #) # test call, where meat of the workload gets done - fre.cmor.cmor_run_subtool( + cmor_run_subtool( indir = INDIR, json_var_list = VARLIST_DIFF, json_table_config = TABLE_CONFIG, diff --git a/run_test_file_cases.py b/run_test_file_cases.py index 92ae40c5..ad34d870 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -9,9 +9,7 @@ import os from pathlib import Path -import fre -from fre.cmor.cmor_mixer import cmor_run_subtool as run_cmor - +from fre.cmor import cmor_run_subtool def print_cwd(): print(f'os.getcwd() = {os.getcwd()}') @@ -40,7 +38,7 @@ def print_the_outcome(some_return,case_str): def run_cmor_RUN(filename, table, opt_var_name): func_debug1 = False if func_debug1: - print('run_cmor(\n' + print('cmor_run_subtool(\n' f' indir = \"{str(Path(filename).parent)}\",\n' f' json_var_list = \"{CMORBITE_VARLIST}\",\n' f' json_table_config = \"{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_{table}.json\",\n' @@ -59,7 +57,7 @@ def run_cmor_RUN(filename, table, opt_var_name): f'-o {os.getcwd()} ' f'-v {opt_var_name} ' ) - FOO_return = run_cmor( + FOO_return = cmor_run_subtool( indir = str(Path(filename).parent), json_var_list = CMORBITE_VARLIST, json_table_config = f'{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_{table}.json', @@ -71,29 +69,29 @@ def run_cmor_RUN(filename, table, opt_var_name): - -#### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. -#### for this case, that file is: -#### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -#### 'pp/ocean_monthly/' + \ -#### 'ocean_monthly.static.nc' -#### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". -# 6) FAIL -# ocean, Omon / sos -# Result - error, it wants lat/lon, but only xh, yh coordinates are available -testfile_ocean_monthly_gn = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly/ts/monthly/5yr/' + \ - 'ocean_monthly.002101-002512.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'ocean_monthly_gn / sos') - -sys.exit() +if False: + #### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. + #### for this case, that file is: + #### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + #### 'pp/ocean_monthly/' + \ + #### 'ocean_monthly.static.nc' + #### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". + # 6) FAIL + # ocean, Omon / sos + # Result - error, it wants lat/lon, but only xh, yh coordinates are available + testfile_ocean_monthly_gn = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly/ts/monthly/5yr/' + \ + 'ocean_monthly.002101-002512.sos.nc' + try: + some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'ocean_monthly_gn / sos') + + sys.exit() From 54e02e31dcb8cc6c7cb8ebf72d28fca1f9865850 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 7 Jan 2025 15:05:01 -0500 Subject: [PATCH 29/45] carefully tidy print statements, carefully changing order of certain things --- fre/cmor/cmor_mixer.py | 274 ++++++++++++++++------------------------- 1 file changed, 105 insertions(+), 169 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 53bd3989..3d31e83b 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -30,24 +30,22 @@ def from_dis_gimme_dis(from_dis, gimme_dis): try: return from_dis[gimme_dis][:].copy() except Exception as exc: - print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}' - f' from this: {from_dis} ' - f' exc = {exc}' - f' returning None!' ) + print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}\n' + #f' from this: {from_dis} \n' + #f' exc = {exc}\n' + ' returning None!\n' ) return None def find_statics_file(bronx_file_path): - print('(find_statics_file) HELLO WORLD!') - #assert type(bronx_file_path) == "" - bronx_file_path_elem=bronx_file_path.split('/') - num_elem=len(bronx_file_path_elem) - print(f'bronx_file_path_elem = {bronx_file_path_elem}') + bronx_file_path_elem = bronx_file_path.split('/') + num_elem = len(bronx_file_path_elem) + print(f'(find_statics_file) bronx_file_path_elem = \n{bronx_file_path_elem}\n') while bronx_file_path_elem[num_elem-2] != 'pp': bronx_file_path_elem.pop() - num_elem=num_elem-1 - print(bronx_file_path_elem) - statics_path='/'.join(bronx_file_path_elem) - statics_file=glob.glob(statics_path+'/*static*.nc')[0] + num_elem = num_elem-1 + #print(bronx_file_path_elem) + statics_path = '/'.join(bronx_file_path_elem) + statics_file = glob.glob(statics_path+'/*static*.nc')[0] if Path(statics_file).exists(): return statics_file else: @@ -65,8 +63,8 @@ def create_lev_bnds(bound_these = None, with_these = None): the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) for i in range(0,len(bound_these)): - the_bnds[i][0]=with_these[i] - the_bnds[i][1]=with_these[i+1] + the_bnds[i][0] = with_these[i] + the_bnds[i][1] = with_these[i+1] print(f'(create_lev_bnds) the_bnds is... ') print(f' the_bnds = \n{the_bnds}') return the_bnds @@ -82,13 +80,13 @@ def get_var_filenames(indir, var_filenames = None, local_var = None): ''' if var_filenames is None: var_filenames = [] - filename_pattern='.nc' if local_var is None else f'.{local_var}.nc' - print(f'(get_var_filenames) filename_pattern={filename_pattern}') - var_filenames_all=glob.glob(f'{indir}/*{filename_pattern}') - print(f'(get_var_filenames) var_filenames_all={var_filenames_all}') + filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc' + print(f'(get_var_filenames) filename_pattern = {filename_pattern}\n') + var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}') + #print(f'(get_var_filenames) var_filenames_all = {var_filenames_all}') for var_file in var_filenames_all: var_filenames.append( Path(var_file).name ) - print(f"(get_var_filenames) var_filenames = {var_filenames}") + #print(f"(get_var_filenames) var_filenames = {var_filenames}") if len(var_filenames) < 1: raise ValueError(f'target directory had no files with .nc ending. indir =\n {indir}') var_filenames.sort() @@ -123,8 +121,11 @@ def check_dataset_for_ocean_grid(ds): ds: netCDF4.Dataset object containing variables with associated dimensional information. ''' if "xh" in list(ds.variables.keys()): - print("(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd" - " sometimes i don't cmorize right! check me!") + print("\n----------------------------------------------------------------------------------\n" + "(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd\n" + " sometimes i don't cmorize right! check me!\n" + "----------------------------------------------------------------------------------\n" + ) return True return False @@ -187,12 +188,12 @@ def create_tmp_dir(outdir, json_exp_config = None): # once we know where the tmp_dir should be, create it try: - os.makedirs(tmp_dir, exist_ok=True) + os.makedirs(tmp_dir, exist_ok = True) # and if we need to additionally create outdir_from_exp_config... try doing that too if outdir_from_exp_config is not None: print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') try: - os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok=True) + os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok = True) except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too. print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work') print( ' .... oh well! it was ust to try to avoid a warning anyways.... moving on') @@ -210,7 +211,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, netcdf_file = None, target_var = None, json_exp_config = None, - json_table_config = None, prev_path=None, + json_table_config = None, prev_path = None, ):#, tmp_dir = None ): ''' rewrite the input netcdf file nc_fl containing target_var in a CMIP-compliant manner. @@ -244,11 +245,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ' ... this is gonna be fun!' ) # try to read what coordinate(s) we're going to be expecting for the variable - expected_mip_coord_dims=None + expected_mip_coord_dims = None try: expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"] print( '(rewrite_netcdf_file_var) i am hoping to find data for the following coordinate dimensions:\n' - f' expected_mip_coord_dims = {expected_mip_coord_dims}' ) + f' expected_mip_coord_dims = {expected_mip_coord_dims}\n' ) except Exception as exc: print(f'(rewrite_netcdf_file_var) WARNING could not get expected coordinate dimensions for {target_var}. ' ' in proj_table_vars file {json_table_config}. \n exc = {exc}') @@ -287,20 +288,43 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}') var = from_dis_gimme_dis( from_dis = ds, gimme_dis = target_var ) - #var = ds[target_var][:] - # the tripolar grid is designed to reduce distortions in ocean data brought on - # by singularities (poles) being placed in oceans (e.g. the N+S poles of standard sphere grid) - # but, the tripolar grid is complex, so the values stored in the file are a lat/lon *on the tripolar grid* - # in order to get spherical lat/lon, one would need to convert on the fly, but implementing such an inverse is not trivial - # thankfully, the spherical lat/lons tend to already be computed in advance, and stored elsewhere. at GFDL they're in "statics" - do_special_ocean_file_stuff=all( [ uses_ocean_grid, + + # grab var_dim + var_dim = len(var.shape) + print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") + + ## Check var_dim + #if var_dim not in [3, 4]: + # raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") + + # determine the vertical dimension by looping over netcdf variables + vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present + print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") + + # Check var_dim and vert_dim and assign lev if relevant. + # error if vert_dim wrong given var_dim + lev, lev_units = None, "1" #1 #"none" #None #"" + lev_bnds = None + if vert_dim != 0: + if vert_dim.lower() not in [ "z_l", "landuse", "plev39", "plev30", "plev19", "plev8", + "height2m", "level", "lev", "levhalf"] : + raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') + lev = ds[vert_dim] + if vert_dim.lower() != "landuse": + lev_units = ds[vert_dim].units + + + + # the tripolar grid is designed to reduce distortions in ocean data brought on + # by singularities (poles) being placed in oceans + # the spherical lat/lons tend to already be computed in advance at GFDL, they're in "statics" + do_special_ocean_file_stuff = all( [ uses_ocean_grid, lat is None, lon is None ] ) - statics_file_path = None x, y = None, None - i_ind, j_ind = None, None - cmor_grid_id = None + cmor_x, cmor_y = None, None + vertex = None if do_special_ocean_file_stuff: try: print(f'(rewrite_netcdf_file_var) netcdf_file is {netcdf_file}') @@ -310,34 +334,36 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) WARNING: pretty sure an ocean statics file is needed, but it could not be found.' ' moving on and doing my best, but i am probably going to break' ) raise Exception('(rewrite_netcdf_file_var) EXITING BC STATICS') from exc + print(f"(rewrite_netcdf_file_var) statics file found.") - statics_file_name=Path(statics_file_path).name - put_statics_file_here=str(Path(netcdf_file).parent) + statics_file_name = Path(statics_file_path).name + put_statics_file_here = str(Path(netcdf_file).parent) shutil.copy(statics_file_path, put_statics_file_here) del statics_file_path + statics_file_path = put_statics_file_here + '/' + statics_file_name print(f'(rewrite_netcdf_file_var) statics file path is now: {statics_file_path}') - statics_ds=nc.Dataset(statics_file_path, 'r') + statics_ds = nc.Dataset(statics_file_path, 'r') # grab the lat/lon points, have shape (yh, xh) statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy() statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy() - print(f'FOO min entry of geolat: {statics_lat[:].data.min()}') - print(f'BAR min entry of geolon: {statics_lon[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of geolat: {statics_lat[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of geolon: {statics_lon[:].data.min()}\n') lat = ds.createVariable('lat', np.float32, ('yh', 'xh') ) lat[:] = statics_lat[:] lon = ds.createVariable('lon', np.float32, ('yh', 'xh') ) lon[:] = statics_lon[:] - print(f'FOO min entry of lat: {lat[:].data.min()}') - print(f'BAR min entry of lon: {lon[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of lat: {lat[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of lon: {lon[:].data.min()}\n') # grab the corners of the cells, should have shape (yh+1, xh+1) lat_c = from_dis_gimme_dis(statics_ds,'geolat_c') lon_c = from_dis_gimme_dis(statics_ds,'geolon_c') - print(f'FOO min entry of geolat_c: {lat_c[:].data.min()}') - print(f'BAR min entry of geolon_c: {lon_c[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of geolat_c: {lat_c[:].data.min()}') + print(f'(rewrite_netcdf_file_var) min entry of geolon_c: {lon_c[:].data.min()}\n') vertex = 4 ds.createDimension('vertex', vertex) @@ -356,75 +382,14 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner - print(f'(rewrite_netcdf_file_var) HARD PART: creating indices (j_index) from y (yh)') + print(f'(rewrite_netcdf_file_var) HARD PART: (yh)') y = from_dis_gimme_dis(ds, 'yh') - print(f' ds.createVariable...') - #j_ind = ds.createVariable('j', int, ('yh') ) - j_ind = ds.createVariable('j_index', np.int32, ('yh') ) - print(f' np.arange...') - #j_ind[:] = np.zeros(len(y), dtype=int ) - j_ind[:] = np.arange(0, len(y), dtype=np.int32 ) - - - print(f'(rewrite_netcdf_file_var) HARD PART: creating indices (i_index) from x (xh)') + print(f'(rewrite_netcdf_file_var) HARD PART: (xh)\n') x = from_dis_gimme_dis(ds, 'xh') - print(f' ds.createVariable...') - #i_ind = ds.createVariable('i', int, ('xh') ) - i_ind = ds.createVariable('i_index', np.int32, ('xh') ) - print(f' np.arange...') - #i_ind[:] = np.zeros(len(x), dtype=int ) - i_ind[:] = np.arange(0, len(x), dtype=np.int32 ) - - #cmor_grid_id = cmor.grid( ) - - #var.coordinates = 'lat lon' - var.coordinates = 'j_index i_index' - #var.coordinates = '' - - - - - - - - - - #print(f' geolat = {lat}') - #assert False - - - - - - - - - # grab var_dim - var_dim = len(var.shape) - print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}") - - ## Check var_dim - #if var_dim not in [3, 4]: - # raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.") - - # determine the vertical dimension by looping over netcdf variables - vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present - print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - - # Check var_dim and vert_dim and assign lev if relevant. - # error if vert_dim wrong given var_dim - lev, lev_units = None, "1" #1 #"none" #None #"" - lev_bnds = None - if vert_dim != 0: - if vert_dim.lower() not in [ "z_l", "landuse", "plev39", "plev30", "plev19", "plev8", - "height2m", "level", "lev", "levhalf"] : - raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') - lev = ds[vert_dim] - if vert_dim.lower() != "landuse": - lev_units = ds[vert_dim].units + # now we set up the cmor module object # initialize CMOR cmor.setup( @@ -439,25 +404,26 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f"(rewrite_netcdf_file_var) cmor is opening: json_exp_config = {json_exp_config}") cmor.dataset_json(json_exp_config) - # load CMOR table - print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") + # if we need to do, e.g. cmor_grid, do that here, before the rest of the routine if do_special_ocean_file_stuff: - print("FOOOOOOOOOOOOOOOOOOOOOOO"+ str(Path(json_table_config).parent) + '/CMIP6_grids.json') + print('(rewrite_netcdf_file_var) SPECIAL OCEAN STUFF opening' + str(Path(json_table_config).parent) + '/CMIP6_grids.json') cmor.load_table( str(Path(json_table_config).parent) + '/CMIP6_grids.json' ) - else: - cmor.load_table(json_table_config) + + #assert False, 'DEBUG STOP AND CHECK' + + # load CMOR table + print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") + cmor.load_table(json_table_config) units = proj_table_vars["variable_entry"] [target_var] ["units"] - print(f"(rewrite_netcdf_file_var) units={units}") + print(f"(rewrite_netcdf_file_var) units = {units}") # setup cmor latitude axis if relevant cmor_lat = None if do_special_ocean_file_stuff: - print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for an index!') - #cmor_lat = cmor.axis("j", coord_vals = j_ind[:], units = "1") - cmor_lat = cmor.axis("j_index", coord_vals = j_ind[:], units = "1") - #cmor_lat = cmor.axis("projection_y_coordinate", coord_vals = y[:], units = "degrees") + print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!') + cmor_lat = cmor.axis("projection_y_coordinate", coord_vals = y[:], units = "degrees") elif any( [ lat is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') else: @@ -471,10 +437,8 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor longitude axis if relevant cmor_lon = None if do_special_ocean_file_stuff: - print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for an index!') - #cmor_lon = cmor.axis("i", coord_vals = i_ind[:], units = "1") - cmor_lon = cmor.axis("i_index", coord_vals = i_ind[:], units = "1") - #cmor_lon = cmor.axis("projection_x_coordinate", coord_vals = x[:], units = "degrees") + print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!') + cmor_lon = cmor.axis("projection_x_coordinate", coord_vals = x[:], units = "degrees") elif any( [ lon is None ] ): print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') else: @@ -483,17 +447,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f' DONE assigning cmor_lon') - # setup the cmor_grid when needed (ocean things, typically) - cmor_grid = None - if do_special_ocean_file_stuff: - cmor_grid = cmor.grid([cmor_lat, cmor_lon], - latitude = lat[:], longitude = lon[:], - latitude_vertices = lat_bnds[:], - longitude_vertices = lon_bnds[:]) - - # load back up the normal table file? - cmor.load_table(json_table_config) - # setup cmor time axis if relevant cmor_time = None print(f'(rewrite_netcdf_file_var) assigning cmor_time') @@ -511,23 +464,6 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) print(f' DONE assigning cmor_time') -# # setup cmor time axis if relevant -# cmor_time = None -# try: -# print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" -# f" coord_vals = \n{time_coords}, \n" -# f" cell_bounds = time_bnds, units = {time_coord_units}) ") -# print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') -# cmor_time = cmor.axis("time", coord_vals = time_coords, -# cell_bounds = time_bnds, units = time_coord_units) -# except ValueError as exc: -# print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n" -# " cmor_time = cmor.axis('time', \n" -# " coord_vals = time_coords, units = time_coord_units)") -# print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') -# cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - - # other vertical-axis-relevant initializations save_ps = False @@ -735,18 +671,16 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, # TODO think of better way to write this kind of conditional data movement... # now we have a file in our targets, point CMOR to the configs and the input file(s) - make_cmor_write_here = None - print( Path( tmp_dir ) ) - print( Path( os.getcwd() ) ) + make_cmor_write_here = None #print( Path( tmp_dir ) ) #print( Path( os.getcwd() ) ) if Path( tmp_dir ).is_absolute(): - print(f'tmp_dir is absolute') + #print(f'tmp_dir is absolute') make_cmor_write_here = tmp_dir elif Path( tmp_dir ).exists(): # relative to where we are - print(f'tmp_dir is relative to CWD!') + #print(f'tmp_dir is relative to CWD!') make_cmor_write_here = os.getcwd() + '/'+tmp_dir # unavoidable, cmor module FORCES write to CWD assert make_cmor_write_here is not None - gotta_go_back_here=os.getcwd() + gotta_go_back_here = os.getcwd() try: print(f"(cmorize_target_var_files) WARNING changing directory to: \n {make_cmor_write_here}" ) os.chdir( make_cmor_write_here ) @@ -772,8 +706,8 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, # now that CMOR has rewritten things... we can take our post-rewriting actions # the final output filename will be... - print(f'(cmorize_target_var_files) local_file_name={local_file_name}') - filename =f"{outdir}/{local_file_name}" + print(f'(cmorize_target_var_files) local_file_name = {local_file_name}') + filename = f"{outdir}/{local_file_name}" print(f"(cmorize_target_var_files) filename = {filename}") # the final output file directory will be... @@ -851,11 +785,11 @@ def cmor_run_subtool( indir = None, ' {json_exp_config}, {outdir}]' ) # open CMOR table config file - print( '(cmor_run_subtool) getting table variables from json_table_config = \n' - f' {json_table_config}' ) + print( '(cmor_run_subtool) loading json_table_config = \n' + f' {json_table_config}\n' ) try: with open( json_table_config, "r", encoding = "utf-8") as table_config_file: - proj_table_vars=json.load(table_config_file) + proj_table_vars = json.load(table_config_file) except Exception as exc: raise FileNotFoundError( @@ -866,7 +800,8 @@ def cmor_run_subtool( indir = None, json_table_config= str( Path(json_table_config).resolve() ) # open input variable list - print('(cmor_run_subtool) opening variable list json_var_list') + print('(cmor_run_subtool) loading json_var_list = \n ' + ' {var_list_file}\n') try: with open( json_var_list, "r", encoding = "utf-8" ) as var_list_file: var_list = json.load( var_list_file ) @@ -896,12 +831,13 @@ def cmor_run_subtool( indir = None, if all( [ opt_var_name is not None, local_var != opt_var_name ] ): - print(f'(cmor_run_subtool) WARNING: skipping local_var={local_var} as it is not equal\n' + print(f'(cmor_run_subtool) WARNING: skipping local_var = {local_var} as it is not equal\n' ' to the opt_var_name argument.') continue - + print('\n') + # it is in there, get the name of the data inside the netcdf file. - target_var=var_list[local_var] # often equiv to local_var but not necessarily. + target_var = var_list[local_var] # often equiv to local_var but not necessarily. if local_var != target_var: print(f'(cmor_run_subtool) WARNING: local_var == {local_var} \n' f' != {target_var} == target_var\n' @@ -912,12 +848,12 @@ def cmor_run_subtool( indir = None, # examine input directory to obtain a list of input file targets var_filenames = [] get_var_filenames(indir, var_filenames, local_var) - print(f"(cmor_run_subtool) found filenames = \n {var_filenames}") + print(f"(cmor_run_subtool) found filenames = \n {var_filenames}\n") # examine input files to obtain target date ranges iso_datetime_arr = [] get_iso_datetimes(var_filenames, iso_datetime_arr) - print(f"(cmor_run_subtool) found iso datetimes = \n {iso_datetime_arr}") + print(f"(cmor_run_subtool) found iso datetimes = \n {iso_datetime_arr}\n") # name_of_set == component label... # which is not relevant for CMOR/CMIP... or is it? From d6013a69e994662a13e19bdc3f5f442e0b960dbe Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 9 Jan 2025 17:13:45 -0500 Subject: [PATCH 30/45] new cmor_helpers file holds helper functions cmor_mixer now creates xh and yh bnds. this was not supposed to take so long... why? see note-to-self... ...note to self, if you need to write data to a netcdf file... don't use [x][y][z] style indexing. you HAVE to use [x,y,z] for the assignment or the assignment fails. --- fre/cmor/cmor_helpers.py | 206 +++++++++++++++++++++++ fre/cmor/cmor_mixer.py | 353 ++++++++++++++------------------------- run_test_file_cases.py | 261 +++++++++++++++-------------- 3 files changed, 464 insertions(+), 356 deletions(-) create mode 100644 fre/cmor/cmor_helpers.py diff --git a/fre/cmor/cmor_helpers.py b/fre/cmor/cmor_helpers.py new file mode 100644 index 00000000..c4707c6d --- /dev/null +++ b/fre/cmor/cmor_helpers.py @@ -0,0 +1,206 @@ +import os +import json +import glob +from pathlib import Path +import numpy as np + + +def print_data_minmax(ds_variable = None, desc = None): + ''' + prints the min/max of numpy.ma.core.MaskedArray (ds_variable) and the name/description (desc) of the data + ''' + print('----------------------------------------------------------------------------------------------------------') + try: + print(f'(print_data_minmax) info for \n' + f' desc = {desc} \n {type(ds_variable)}') + #print(f'(print_data_minmax) {ds_variable.data.min()} < {desc} < {ds_variable.data.max()}') + print(f'(print_data_minmax) {ds_variable.min()} < {desc} < {ds_variable.max()}') + except: + print(f'(print_data_minmax) could not print min/max entries for desc = {desc}') + pass + print('----------------------------------------------------------------------------------------------------------') + return + + +def from_dis_gimme_dis(from_dis, gimme_dis): + ''' + gives you gimme_dis from from_dis. accepts two arguments, both mandatory. + from_dis: the target netCDF4.Dataset object to try reading from + gimme_dis: what from_dis is hopefully gonna have and you're gonna get + ''' + try: + return from_dis[gimme_dis][:].copy() + except Exception as exc: + print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}\n' + #f' from this: {from_dis} \n' + #f' exc = {exc}\n' + ' returning None!\n' ) + return None + +def find_statics_file(bronx_file_path): + bronx_file_path_elem = bronx_file_path.split('/') + num_elem = len(bronx_file_path_elem) + print(f'(find_statics_file) bronx_file_path_elem = \n{bronx_file_path_elem}\n') + while bronx_file_path_elem[num_elem-2] != 'pp': + bronx_file_path_elem.pop() + num_elem = num_elem-1 + #print(bronx_file_path_elem) + statics_path = '/'.join(bronx_file_path_elem) + statics_file = glob.glob(statics_path+'/*static*.nc')[0] + if Path(statics_file).exists(): + return statics_file + else: + return None + + +def create_lev_bnds(bound_these = None, with_these = None): + the_bnds = None + assert len(with_these) == len(bound_these) + 1 + print(f'(create_lev_bnds) bound_these is... ') + print(f' bound_these = \n{bound_these}') + print(f'(create_lev_bnds) with_these is... ') + print(f' with_these = \n{with_these}') + + + the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) + for i in range(0,len(bound_these)): + the_bnds[i][0] = with_these[i] + the_bnds[i][1] = with_these[i+1] + print(f'(create_lev_bnds) the_bnds is... ') + print(f' the_bnds = \n{the_bnds}') + return the_bnds + +def get_var_filenames(indir, var_filenames = None, local_var = None): + ''' + appends files ending in .nc located within indir to list var_filenames accepts three arguments + indir: string, representing a path to a directory containing files ending in .nc extension + var_filenames: list of strings, empty or non-empty, to append discovered filenames to. the + object pointed to by the reference var_filenames is manipulated, and so need + not be returned. + local_var: string, optional, if not None, will be used for ruling out filename targets + ''' + if var_filenames is None: + var_filenames = [] + filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc' + print(f'(get_var_filenames) filename_pattern = {filename_pattern}\n') + var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}') + #print(f'(get_var_filenames) var_filenames_all = {var_filenames_all}') + for var_file in var_filenames_all: + var_filenames.append( Path(var_file).name ) + #print(f"(get_var_filenames) var_filenames = {var_filenames}") + if len(var_filenames) < 1: + raise ValueError(f'target directory had no files with .nc ending. indir =\n {indir}') + var_filenames.sort() + + +def get_iso_datetimes(var_filenames, iso_datetime_arr = None): + ''' + appends iso datetime strings found amongst filenames to iso_datetime_arr. + var_filenames: non-empty list of strings representing filenames. some of which presumably + contain datetime strings + iso_datetime_arr: list of strings, empty or non-empty, representing datetimes found in + var_filenames entries. the objet pointed to by the reference + iso_datetime_arr is manipulated, and so need-not be returned + ''' + if iso_datetime_arr is None: + iso_datetime_arr = [] + for filename in var_filenames: + iso_datetime = filename.split(".")[1] + if iso_datetime not in iso_datetime_arr: + iso_datetime_arr.append( + filename.split(".")[1] ) + iso_datetime_arr.sort() + #print(f"(get_iso_datetimes) Available dates: {iso_datetime_arr}") + if len(iso_datetime_arr) < 1: + raise ValueError('(get_iso_datetimes) ERROR: iso_datetime_arr has length 0!') + + +def check_dataset_for_ocean_grid(ds): + ''' + checks netCDF4.Dataset ds for ocean grid origin, and throws an error if it finds one. accepts + one argument. this function has no return. + ds: netCDF4.Dataset object containing variables with associated dimensional information. + ''' + if "xh" in list(ds.variables.keys()): + print("\n----------------------------------------------------------------------------------\n" + "(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd\n" + " sometimes i don't cmorize right! check me!\n" + "----------------------------------------------------------------------------------\n" + ) + return True + return False + + + +def get_vertical_dimension(ds, target_var): + ''' + determines the vertical dimensionality of target_var within netCDF4 Dataset ds. accepts two + arguments and returns an object represnting the vertical dimensions assoc with the target_var. + ds: netCDF4.Dataset object containing variables with associated dimensional information. + target_var: string, representating a variable contained within the netCDF4.Dataset ds + ''' + vert_dim = 0 + for name, variable in ds.variables.items(): + if name != target_var: # not the var we are looking for? move on. + continue + dims = variable.dimensions + for dim in dims: #print(f'(get_vertical_dimension) dim={dim}') + + # check for special case + if dim.lower() == 'landuse': # aux coordinate, so has no axis property + vert_dim = dim + break + + # if it is not a vertical axis, move on. + if not (ds[dim].axis and ds[dim].axis == "Z"): + continue + vert_dim = dim + + return vert_dim + +def create_tmp_dir(outdir, json_exp_config = None): + ''' + creates a tmp_dir based on targeted output directory root. returns the name of the tmp dir. + accepts one argument: + outdir: string, representing the final output directory root for the cmor modules netcdf + file output. tmp_dir will be slightly different depending on the output directory + targeted + ''' + # first see if the exp_config has any additional output path structure to create + outdir_from_exp_config = None + if json_exp_config is not None: + with open(json_exp_config, "r", encoding = "utf-8") as table_config_file: + try: + outdir_from_exp_config = json.load(table_config_file)["outpath"] + except: + print(f'(create_tmp_dir) WARNING could not read outdir from json_exp_config.' + ' the cmor module will throw a toothless warning' ) + + # assign an appropriate temporary working directory + tmp_dir = None + if any( [ outdir == "/local2", + outdir.find("/work") != -1, + outdir.find("/net" ) != -1 ] ): + tmp_dir = str( Path("{outdir}/").resolve() ) + print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {tmp_dir} )') + else: + tmp_dir = str( Path(f"{outdir}/tmp/").resolve() ) + print(f'(create_tmp_dir) NOT using /local /work /net ( tmp_dir = {tmp_dir} )') + + # once we know where the tmp_dir should be, create it + try: + os.makedirs(tmp_dir, exist_ok = True) + # and if we need to additionally create outdir_from_exp_config... try doing that too + if outdir_from_exp_config is not None: + print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') + try: + os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok = True) + except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too. + print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work') + print( ' .... oh well! it was ust to try to avoid a warning anyways.... moving on') + pass + except Exception as exc: + raise OSError(f'(create_tmp_dir) problem creating tmp output directory {tmp_dir}. stop.') from exc + + return tmp_dir + diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 3d31e83b..603674d9 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -5,7 +5,6 @@ ''' import os -import glob import json import shutil import subprocess @@ -15,196 +14,13 @@ import netCDF4 as nc import cmor +from .cmor_helpers import * -# ----- \start consts +# ----- \start consts # TODO make this an input argument flag or smth. DEBUG_MODE_RUN_ONE = True # ----- \end consts -def from_dis_gimme_dis(from_dis, gimme_dis): - ''' - gives you gimme_dis from from_dis. accepts two arguments, both mandatory. - from_dis: the target netCDF4.Dataset object to try reading from - gimme_dis: what from_dis is hopefully gonna have and you're gonna get - ''' - try: - return from_dis[gimme_dis][:].copy() - except Exception as exc: - print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}\n' - #f' from this: {from_dis} \n' - #f' exc = {exc}\n' - ' returning None!\n' ) - return None - -def find_statics_file(bronx_file_path): - bronx_file_path_elem = bronx_file_path.split('/') - num_elem = len(bronx_file_path_elem) - print(f'(find_statics_file) bronx_file_path_elem = \n{bronx_file_path_elem}\n') - while bronx_file_path_elem[num_elem-2] != 'pp': - bronx_file_path_elem.pop() - num_elem = num_elem-1 - #print(bronx_file_path_elem) - statics_path = '/'.join(bronx_file_path_elem) - statics_file = glob.glob(statics_path+'/*static*.nc')[0] - if Path(statics_file).exists(): - return statics_file - else: - return None - - -def create_lev_bnds(bound_these = None, with_these = None): - the_bnds = None - assert len(with_these) == len(bound_these) + 1 - print(f'(create_lev_bnds) bound_these is... ') - print(f' bound_these = \n{bound_these}') - print(f'(create_lev_bnds) with_these is... ') - print(f' with_these = \n{with_these}') - - - the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2) - for i in range(0,len(bound_these)): - the_bnds[i][0] = with_these[i] - the_bnds[i][1] = with_these[i+1] - print(f'(create_lev_bnds) the_bnds is... ') - print(f' the_bnds = \n{the_bnds}') - return the_bnds - -def get_var_filenames(indir, var_filenames = None, local_var = None): - ''' - appends files ending in .nc located within indir to list var_filenames accepts three arguments - indir: string, representing a path to a directory containing files ending in .nc extension - var_filenames: list of strings, empty or non-empty, to append discovered filenames to. the - object pointed to by the reference var_filenames is manipulated, and so need - not be returned. - local_var: string, optional, if not None, will be used for ruling out filename targets - ''' - if var_filenames is None: - var_filenames = [] - filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc' - print(f'(get_var_filenames) filename_pattern = {filename_pattern}\n') - var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}') - #print(f'(get_var_filenames) var_filenames_all = {var_filenames_all}') - for var_file in var_filenames_all: - var_filenames.append( Path(var_file).name ) - #print(f"(get_var_filenames) var_filenames = {var_filenames}") - if len(var_filenames) < 1: - raise ValueError(f'target directory had no files with .nc ending. indir =\n {indir}') - var_filenames.sort() - - -def get_iso_datetimes(var_filenames, iso_datetime_arr = None): - ''' - appends iso datetime strings found amongst filenames to iso_datetime_arr. - var_filenames: non-empty list of strings representing filenames. some of which presumably - contain datetime strings - iso_datetime_arr: list of strings, empty or non-empty, representing datetimes found in - var_filenames entries. the objet pointed to by the reference - iso_datetime_arr is manipulated, and so need-not be returned - ''' - if iso_datetime_arr is None: - iso_datetime_arr = [] - for filename in var_filenames: - iso_datetime = filename.split(".")[1] - if iso_datetime not in iso_datetime_arr: - iso_datetime_arr.append( - filename.split(".")[1] ) - iso_datetime_arr.sort() - #print(f"(get_iso_datetimes) Available dates: {iso_datetime_arr}") - if len(iso_datetime_arr) < 1: - raise ValueError('(get_iso_datetimes) ERROR: iso_datetime_arr has length 0!') - - -def check_dataset_for_ocean_grid(ds): - ''' - checks netCDF4.Dataset ds for ocean grid origin, and throws an error if it finds one. accepts - one argument. this function has no return. - ds: netCDF4.Dataset object containing variables with associated dimensional information. - ''' - if "xh" in list(ds.variables.keys()): - print("\n----------------------------------------------------------------------------------\n" - "(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd\n" - " sometimes i don't cmorize right! check me!\n" - "----------------------------------------------------------------------------------\n" - ) - return True - return False - - - -def get_vertical_dimension(ds, target_var): - ''' - determines the vertical dimensionality of target_var within netCDF4 Dataset ds. accepts two - arguments and returns an object represnting the vertical dimensions assoc with the target_var. - ds: netCDF4.Dataset object containing variables with associated dimensional information. - target_var: string, representating a variable contained within the netCDF4.Dataset ds - ''' - vert_dim = 0 - for name, variable in ds.variables.items(): - if name != target_var: # not the var we are looking for? move on. - continue - dims = variable.dimensions - for dim in dims: #print(f'(get_vertical_dimension) dim={dim}') - - # check for special case - if dim.lower() == 'landuse': # aux coordinate, so has no axis property - vert_dim = dim - break - - # if it is not a vertical axis, move on. - if not (ds[dim].axis and ds[dim].axis == "Z"): - continue - vert_dim = dim - - return vert_dim - -def create_tmp_dir(outdir, json_exp_config = None): - ''' - creates a tmp_dir based on targeted output directory root. returns the name of the tmp dir. - accepts one argument: - outdir: string, representing the final output directory root for the cmor modules netcdf - file output. tmp_dir will be slightly different depending on the output directory - targeted - ''' - # first see if the exp_config has any additional output path structure to create - outdir_from_exp_config = None - if json_exp_config is not None: - with open(json_exp_config, "r", encoding = "utf-8") as table_config_file: - try: - outdir_from_exp_config = json.load(table_config_file)["outpath"] - except: - print(f'(create_tmp_dir) WARNING could not read outdir from json_exp_config.' - ' the cmor module will throw a toothless warning' ) - - # assign an appropriate temporary working directory - tmp_dir = None - if any( [ outdir == "/local2", - outdir.find("/work") != -1, - outdir.find("/net" ) != -1 ] ): - tmp_dir = str( Path("{outdir}/").resolve() ) - print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {tmp_dir} )') - else: - tmp_dir = str( Path(f"{outdir}/tmp/").resolve() ) - print(f'(create_tmp_dir) NOT using /local /work /net ( tmp_dir = {tmp_dir} )') - - # once we know where the tmp_dir should be, create it - try: - os.makedirs(tmp_dir, exist_ok = True) - # and if we need to additionally create outdir_from_exp_config... try doing that too - if outdir_from_exp_config is not None: - print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ') - try: - os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok = True) - except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too. - print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work') - print( ' .... oh well! it was ust to try to avoid a warning anyways.... moving on') - pass - except Exception as exc: - raise OSError(f'(create_tmp_dir) problem creating tmp output directory {tmp_dir}. stop.') from exc - - return tmp_dir - - - ### ------ BULK ROUTINES ------ ### def rewrite_netcdf_file_var ( proj_table_vars = None, local_var = None, @@ -300,7 +116,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # determine the vertical dimension by looping over netcdf variables vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - + # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev, lev_units = None, "1" #1 #"none" #None #"" @@ -322,10 +138,13 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lat is None, lon is None ] ) statics_file_path = None - x, y = None, None - cmor_x, cmor_y = None, None - vertex = None + xh, yh = None, None #cmor_x, cmor_y = None, None + xh_dim, yh_dim = None, None + xh_bnds, yh_bnds = None, None + bnds, vertex = None, None if do_special_ocean_file_stuff: + + # resolve location of statics file required for this processing. try: print(f'(rewrite_netcdf_file_var) netcdf_file is {netcdf_file}') statics_file_path = find_statics_file(prev_path) @@ -344,87 +163,161 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, statics_file_path = put_statics_file_here + '/' + statics_file_name print(f'(rewrite_netcdf_file_var) statics file path is now: {statics_file_path}') + # statics file read statics_ds = nc.Dataset(statics_file_path, 'r') + # grab the lat/lon points, have shape (yh, xh) + print(f'(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell centers from statics file \n') statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy() statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy() - print(f'(rewrite_netcdf_file_var) min entry of geolat: {statics_lat[:].data.min()}') - print(f'(rewrite_netcdf_file_var) min entry of geolon: {statics_lon[:].data.min()}\n') + + print('\n') + print_data_minmax(statics_lat, "statics_lat") + print_data_minmax(statics_lon, "statics_lon") + print('\n') - lat = ds.createVariable('lat', np.float32, ('yh', 'xh') ) + + # spherical lat and lon coords + print(f'(rewrite_netcdf_file_var) creating lat and lon variables in temp file \n') + lat = ds.createVariable('lat', statics_lat.dtype, ('yh', 'xh') ) + lon = ds.createVariable('lon', statics_lon.dtype, ('yh', 'xh') ) lat[:] = statics_lat[:] - lon = ds.createVariable('lon', np.float32, ('yh', 'xh') ) lon[:] = statics_lon[:] - print(f'(rewrite_netcdf_file_var) min entry of lat: {lat[:].data.min()}') - print(f'(rewrite_netcdf_file_var) min entry of lon: {lon[:].data.min()}\n') + + print('\n') + print_data_minmax(lat[:], "lat") + print_data_minmax(lon[:], "lon") + print('\n') + # grab the corners of the cells, should have shape (yh+1, xh+1) + print(f'(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell corners from statics file \n') lat_c = from_dis_gimme_dis(statics_ds,'geolat_c') lon_c = from_dis_gimme_dis(statics_ds,'geolon_c') - print(f'(rewrite_netcdf_file_var) min entry of geolat_c: {lat_c[:].data.min()}') - print(f'(rewrite_netcdf_file_var) min entry of geolon_c: {lon_c[:].data.min()}\n') + + print('\n') + print_data_minmax(lat_c, "lat_c") + print_data_minmax(lon_c, "lon_c") + print('\n') + + # vertex + print(f'(rewrite_netcdf_file_var) creating vertex dimension\n') vertex = 4 ds.createDimension('vertex', vertex) - lat_bnds = ds.createVariable('lat_bnds', np.float32, ('yh', 'xh', 'vertex') ) + + # lat and lon bnds + print(f'(rewrite_netcdf_file_var) creating lat and lon bnds from geolat and geolon of corners\n') + lat_bnds = ds.createVariable('lat_bnds', lat_c.dtype, ('yh', 'xh', 'vertex') ) lat_bnds[:,:,0] = lat_c[1:,1:] # NE corner lat_bnds[:,:,1] = lat_c[1:,:-1] # NW corner lat_bnds[:,:,2] = lat_c[:-1,:-1] # SW corner lat_bnds[:,:,3] = lat_c[:-1,1:] # SE corner - - lon_bnds = ds.createVariable('lon_bnds', np.float32, ('yh', 'xh', 'vertex') ) + lon_bnds = ds.createVariable('lon_bnds', lon_c.dtype, ('yh', 'xh', 'vertex') ) lon_bnds[:,:,0] = lon_c[1:,1:] # NE corner lon_bnds[:,:,1] = lon_c[1:,:-1] # NW corner lon_bnds[:,:,2] = lon_c[:-1,:-1] # SW corner lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner + + print('\n') + print_data_minmax(lat_bnds[:], "lat_bnds") + print_data_minmax(lon_bnds[:], "lon_bnds") + print('\n') + + # grab the h-point lat and lon + print(f'(rewrite_netcdf_file_var) reading yh, xh\n') + yh = from_dis_gimme_dis(ds, 'yh') + xh = from_dis_gimme_dis(ds, 'xh') - print(f'(rewrite_netcdf_file_var) HARD PART: (yh)') - y = from_dis_gimme_dis(ds, 'yh') + print('\n') + print_data_minmax(yh[:], "yh") + print_data_minmax(xh[:], "xh") + print('\n') - print(f'(rewrite_netcdf_file_var) HARD PART: (xh)\n') - x = from_dis_gimme_dis(ds, 'xh') + yh_dim = len(yh) + xh_dim = len(xh) + # read the q-point native-grid lat lon points + print(f'(rewrite_netcdf_file_var) reading yq, xq from statics file \n') + yq = from_dis_gimme_dis(statics_ds, 'yq') + xq = from_dis_gimme_dis(statics_ds, 'xq') - - # now we set up the cmor module object + print('\n') + print_data_minmax(yq, "yq") + print_data_minmax(xq, "xq") + print('\n') + + assert yh_dim == (len(yq)-1) + assert xh_dim == (len(xq)-1) + + # create h-point bounds from the q-point lat lons + print(f'(rewrite_netcdf_file_var) creating yh_bnds, xh_bnds from yq, xq\n') + + yh_bnds = ds.createVariable('yh_bnds', yq.dtype, ( 'yh', 'nv' ) ) + for i in range(0,yh_dim): + yh_bnds[i,0] = yq[i] + yh_bnds[i,1] = yq[i+1] + + xh_bnds = ds.createVariable('xh_bnds', xq.dtype, ( 'xh', 'nv' ) ) + for i in range(0,xh_dim): + xh_bnds[i,0] = xq[i] + xh_bnds[i,1] = xq[i+1] + if i%200 == 0: + print(f' AFTER assignment: xh_bnds[{i}][0] = {xh_bnds[i][0]}') + print(f' AFTER assignment: xh_bnds[{i}][1] = {xh_bnds[i][1]}') + print(f' type(xh_bnds[{i}][1]) = {type(xh_bnds[i][1])}') + + print('\n') + print_data_minmax(yh_bnds[:], "yh_bnds") + print_data_minmax(xh_bnds[:], "xh_bnds") + print('\n') + + + ds.close() + assert False, 'pause+check output' + + # now we set up the cmor module object # initialize CMOR cmor.setup( - netcdf_file_action = cmor.CMOR_PRESERVE, #.CMOR_APPEND,# + netcdf_file_action = cmor.CMOR_APPEND,#.CMOR_PRESERVE, # set_verbosity = cmor.CMOR_QUIET,#.CMOR_NORMAL, # exit_control = cmor.CMOR_NORMAL,#.CMOR_EXIT_ON_WARNING,# -# logfile = './foo.log', + #logfile = './foo.log', create_subdirectories = 1 ) + # read experiment configuration file print(f"(rewrite_netcdf_file_var) cmor is opening: json_exp_config = {json_exp_config}") cmor.dataset_json(json_exp_config) - - # if we need to do, e.g. cmor_grid, do that here, before the rest of the routine - if do_special_ocean_file_stuff: - print('(rewrite_netcdf_file_var) SPECIAL OCEAN STUFF opening' + str(Path(json_table_config).parent) + '/CMIP6_grids.json') - cmor.load_table( str(Path(json_table_config).parent) + '/CMIP6_grids.json' ) - - #assert False, 'DEBUG STOP AND CHECK' - + # load CMOR table - print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}") - cmor.load_table(json_table_config) + print(f"(rewrite_netcdf_file_var) cmor is loading+setting json_table_config = {json_table_config}") + loaded_cmor_table_cfg = cmor.load_table(json_table_config) + cmor.set_table(loaded_cmor_table_cfg) + + + # if ocean tripolar grid, we need the CMIP grids configuration file. load it but don't set the table yet. + json_grids_config, loaded_cmor_grids_cfg = None, None + if do_special_ocean_file_stuff: + print(f'(rewrite_netcdf_file_var) cmor is loading/opening {json_grids_config}') + json_grids_config = str(Path(json_table_config).parent) + '/CMIP6_grids.json' + loaded_cmor_grids_cfg = cmor.load_table( json_grids_config ) + cmor.set_table(loaded_cmor_grids_cfg) - units = proj_table_vars["variable_entry"] [target_var] ["units"] - print(f"(rewrite_netcdf_file_var) units = {units}") + # setup cmor latitude axis if relevant cmor_lat = None if do_special_ocean_file_stuff: print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!') - cmor_lat = cmor.axis("projection_y_coordinate", coord_vals = y[:], units = "degrees") - elif any( [ lat is None ] ): + cmor_lat = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees") + elif lat is None : print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lat') @@ -438,12 +331,16 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, cmor_lon = None if do_special_ocean_file_stuff: print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!') - cmor_lon = cmor.axis("projection_x_coordinate", coord_vals = x[:], units = "degrees") - elif any( [ lon is None ] ): + cmor_lon = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees") + elif lon is None : print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') else: print(f'(rewrite_netcdf_file_var) assigning cmor_lon') cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + if lon_bnds is None: + cmor_lon = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E") + else: + cmor_lon = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E") print(f' DONE assigning cmor_lon') @@ -585,9 +482,13 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f' axes now = {axes}') - # read positive attribute and create cmor_var? can this return none? TODO + # read positive/units attribute and create cmor_var + units = proj_table_vars["variable_entry"] [target_var] ["units"] + print(f"(rewrite_netcdf_file_var) units = {units}") + positive = proj_table_vars["variable_entry"] [target_var] ["positive"] print(f"(rewrite_netcdf_file_var) positive = {positive}") + cmor_var = cmor.variable(target_var, units, axes, positive = positive) # Write the output to disk diff --git a/run_test_file_cases.py b/run_test_file_cases.py index ad34d870..f7586156 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -24,7 +24,7 @@ def print_the_outcome(some_return,case_str): print('-----------------------------------------------------------------------------------------------------------------') print(f'\n\n\n\n\n\n\n\n\n\n') print_cwd() - assert some_return == 0 + #assert some_return == 0 # global consts for these tests, with no/trivial impact on the results ROOTDIR='fre/tests/test_files' @@ -69,7 +69,7 @@ def run_cmor_RUN(filename, table, opt_var_name): -if False: +if True: #### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. #### for this case, that file is: #### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ @@ -91,133 +91,134 @@ def run_cmor_RUN(filename, table, opt_var_name): print_the_outcome(some_return,'ocean_monthly_gn / sos') - sys.exit() - - - -# 1) SUCCEEDs -# land, Lmon, gr1 -testfile_land_gr1_Lmon = \ - '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/land/ts/monthly/5yr/' + \ - 'land.005101-005512.lai.nc' -try: - some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') -except: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'land_gr1_Lmon / lai') - - -# 2) SUCCEEDs -# atmos, Amon / cl -testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.196001-196412.cl.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') - - -# 3) SUCCEEDs -# atmos, Amon / mc -testfile_atmos_level_cmip_gr1_Amon_fullL = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.195501-195912.mc.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') - - -# 4) SUCCEEDs (no longitude coordinate case) -# atmos, AERmonZ / ta -# just like #1, but lack longitude -testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ - 'atmos_plev39_cmip.201001-201412.ta.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') - - -# 5) SUCCEEDs -# ocean, Omon / sos -testfile_ocean_monthly_1x1deg_gr = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_1x1deg.190001-190412.sos.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') - - -# 7) SUCCEEDs -# ocean, Omon / so -testfile_ocean_monthly_z_1x1deg_gr = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -try: - some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') - - -# 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -# atmos, Amon / ch4global -testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' -try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - - -# 9) SUCCEEDs (needs coordinate variable axis with character string values) -# land, Emon / gppLut -testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' -try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - -print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') - - - - + #sys.exit() +if False: + # 1) SUCCEEDs + # land, Lmon, gr1 + testfile_land_gr1_Lmon = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + \ + 'land.005101-005512.lai.nc' + try: + some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') + except: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'land_gr1_Lmon / lai') + + + # 2) SUCCEEDs + # atmos, Amon / cl + testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.196001-196412.cl.nc' + try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') + + + # 3) SUCCEEDs + # atmos, Amon / mc + testfile_atmos_level_cmip_gr1_Amon_fullL = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ + 'atmos_level_cmip.195501-195912.mc.nc' + try: + some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') + + + # 4) SUCCEEDs (no longitude coordinate case) + # atmos, AERmonZ / ta + # just like #1, but lack longitude + testfile_atmos_gr1_AERmonZ_nolons = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ + 'atmos_plev39_cmip.201001-201412.ta.nc' + try: + some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') + + + # 5) SUCCEEDs + # ocean, Omon / sos + testfile_ocean_monthly_1x1deg_gr = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_1x1deg.190001-190412.sos.nc' + try: + some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') + + + # 7) SUCCEEDs + # ocean, Omon / so + testfile_ocean_monthly_z_1x1deg_gr = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ + 'ocean_monthly_z_1x1deg.000101-000512.so.nc' + try: + some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') + + + # 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) + # atmos, Amon / ch4global + testfile_atmos_scalar_gn_Amon_nolon_nolat = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + \ + 'atmos_scalar.197001-197412.ch4global.nc' + try: + some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') + + + + # 9) SUCCEEDs (needs coordinate variable axis with character string values) + # land, Emon / gppLut + testfile_LUmip_refined_gr1_Emon_landusedim = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + \ + 'LUmip_refined.185001-185412.gppLut.nc' + try: + some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') + except Exception as exc: + print(f'exception caught: exc=\n{exc}') + some_return=-1 + + print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') + + + + + + + From 5911351f1d3a15501faebf8d2eed170aadaf60a7 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 9 Jan 2025 17:48:47 -0500 Subject: [PATCH 31/45] SO CLOSE --- fre/cmor/cmor_mixer.py | 127 ++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 60 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 603674d9..ebddc4df 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -134,15 +134,15 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # the tripolar grid is designed to reduce distortions in ocean data brought on # by singularities (poles) being placed in oceans # the spherical lat/lons tend to already be computed in advance at GFDL, they're in "statics" - do_special_ocean_file_stuff = all( [ uses_ocean_grid, + process_tripolar_data = all( [ uses_ocean_grid, lat is None, lon is None ] ) statics_file_path = None xh, yh = None, None #cmor_x, cmor_y = None, None xh_dim, yh_dim = None, None xh_bnds, yh_bnds = None, None - bnds, vertex = None, None - if do_special_ocean_file_stuff: + vertex = None + if process_tripolar_data: # resolve location of statics file required for this processing. try: @@ -277,15 +277,12 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print('\n') - ds.close() - assert False, 'pause+check output' - # now we set up the cmor module object # initialize CMOR cmor.setup( netcdf_file_action = cmor.CMOR_APPEND,#.CMOR_PRESERVE, # set_verbosity = cmor.CMOR_QUIET,#.CMOR_NORMAL, # - exit_control = cmor.CMOR_NORMAL,#.CMOR_EXIT_ON_WARNING,# + exit_control = cmor.CMOR_EXIT_ON_MAJOR,#.CMOR_NORMAL,#.CMOR_EXIT_ON_WARNING,# #logfile = './foo.log', create_subdirectories = 1 ) @@ -303,9 +300,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # if ocean tripolar grid, we need the CMIP grids configuration file. load it but don't set the table yet. json_grids_config, loaded_cmor_grids_cfg = None, None - if do_special_ocean_file_stuff: - print(f'(rewrite_netcdf_file_var) cmor is loading/opening {json_grids_config}') + if process_tripolar_data: json_grids_config = str(Path(json_table_config).parent) + '/CMIP6_grids.json' + print(f'(rewrite_netcdf_file_var) cmor is loading/opening {json_grids_config}') loaded_cmor_grids_cfg = cmor.load_table( json_grids_config ) cmor.set_table(loaded_cmor_grids_cfg) @@ -313,36 +310,46 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # setup cmor latitude axis if relevant - cmor_lat = None - if do_special_ocean_file_stuff: + cmor_y = None + if process_tripolar_data: print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!') - cmor_lat = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees") + cmor_y = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees") elif lat is None : - print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_lat') + print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_y') else: - print(f'(rewrite_netcdf_file_var) assigning cmor_lat') + print(f'(rewrite_netcdf_file_var) assigning cmor_y') if lat_bnds is None: - cmor_lat = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N") + cmor_y = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N") else: - cmor_lat = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N") - print(f' DONE assigning cmor_lat') + cmor_y = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N") + print(f' DONE assigning cmor_y') # setup cmor longitude axis if relevant - cmor_lon = None - if do_special_ocean_file_stuff: + cmor_x = None + if process_tripolar_data: print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!') - cmor_lon = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees") + cmor_x = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees") elif lon is None : - print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_lon') + print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_x') else: - print(f'(rewrite_netcdf_file_var) assigning cmor_lon') - cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") + print(f'(rewrite_netcdf_file_var) assigning cmor_x') + cmor_x = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") if lon_bnds is None: - cmor_lon = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E") + cmor_x = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E") else: - cmor_lon = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E") - print(f' DONE assigning cmor_lon') - + cmor_x = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E") + print(f' DONE assigning cmor_x') + + cmor_grid = None + if process_tripolar_data: + print(f'(rewrite_netcdf_file_var) WARNING setting cmor.grid, process_tripolar_data = {process_tripolar_data}') + cmor_grid = cmor.grid( axis_ids = [cmor_y, cmor_x], + latitude = lat[:], longitude = lon[:], + latitude_vertices = lat_bnds[:], + longitude_vertices = lon_bnds[:] ) + + # now that we are done with setting the grid, we can go back to the usual approach + cmor.set_table(loaded_cmor_table_cfg) # setup cmor time axis if relevant cmor_time = None @@ -369,20 +376,20 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ips = None # set cmor vertical axis if relevant - cmor_lev = None + cmor_z = None if lev is not None: - print(f'(rewrite_netcdf_file_var) assigning cmor_lev') + print(f'(rewrite_netcdf_file_var) assigning cmor_z') if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') if vert_dim.lower() != "landuse": cmor_vert_dim_name = vert_dim - cmor_lev = cmor.axis( cmor_vert_dim_name, + cmor_z = cmor.axis( cmor_vert_dim_name, coord_vals = lev[:], units = lev_units ) else: landuse_str_list=['primary_and_secondary_land', 'pastures', 'crops', 'urban'] cmor_vert_dim_name = "landUse" # this is why can't we have nice things - cmor_lev = cmor.axis( cmor_vert_dim_name, + cmor_z = cmor.axis( cmor_vert_dim_name, coord_vals = np.array( landuse_str_list, dtype=f'S{len(landuse_str_list[0])}' ), @@ -394,7 +401,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, with_these = ds['z_i'] ) print('(rewrite_netcdf_file_var) created lev_bnds...') print(f' lev_bnds = \n{lev_bnds}') - cmor_lev = cmor.axis( 'depth_coord', + cmor_z = cmor.axis( 'depth_coord', coord_vals = lev[:], units = lev_units, cell_bounds = lev_bnds) @@ -408,33 +415,33 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # assign lev_half specifics if vert_dim == "levhalf": - cmor_lev = cmor.axis( "alternate_hybrid_sigma_half", + cmor_z = cmor.axis( "alternate_hybrid_sigma_half", coord_vals = lev[:], units = lev_units ) - ierr_ap = cmor.zfactor( zaxis_id = cmor_lev, + ierr_ap = cmor.zfactor( zaxis_id = cmor_z, zfactor_name = "ap_half", - axis_ids = [cmor_lev, ], + axis_ids = [cmor_z, ], zfactor_values = ds["ap_bnds"][:], units = ds["ap_bnds"].units ) - ierr_b = cmor.zfactor( zaxis_id = cmor_lev, + ierr_b = cmor.zfactor( zaxis_id = cmor_z, zfactor_name = "b_half", - axis_ids = [cmor_lev, ], + axis_ids = [cmor_z, ], zfactor_values = ds["b_bnds"][:], units = ds["b_bnds"].units ) else: - cmor_lev = cmor.axis( "alternate_hybrid_sigma", + cmor_z = cmor.axis( "alternate_hybrid_sigma", coord_vals = lev[:], units = lev_units, cell_bounds = ds[vert_dim+"_bnds"] ) - ierr_ap = cmor.zfactor( zaxis_id = cmor_lev, + ierr_ap = cmor.zfactor( zaxis_id = cmor_z, zfactor_name = "ap", - axis_ids = [cmor_lev, ], + axis_ids = [cmor_z, ], zfactor_values = ds["ap"][:], zfactor_bounds = ds["ap_bnds"][:], units = ds["ap"].units ) - ierr_b = cmor.zfactor( zaxis_id = cmor_lev, + ierr_b = cmor.zfactor( zaxis_id = cmor_z, zfactor_name = "b", - axis_ids = [cmor_lev, ], + axis_ids = [cmor_z, ], zfactor_values = ds["b"][:], zfactor_bounds = ds["b_bnds"][:], units = ds["b"].units ) @@ -446,21 +453,21 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') axis_ids.append(cmor_time) print(f' axis_ids now = {axis_ids}') - if cmor_lat is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lat to axis_ids list...') - axis_ids.append(cmor_lat) + if cmor_y is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_y to axis_ids list...') + axis_ids.append(cmor_y) print(f' axis_ids now = {axis_ids}') - if cmor_lon is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lon to axis_ids list...') - axis_ids.append(cmor_lon) + if cmor_x is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_x to axis_ids list...') + axis_ids.append(cmor_x) print(f' axis_ids now = {axis_ids}') - ips = cmor.zfactor( zaxis_id = cmor_lev, + ips = cmor.zfactor( zaxis_id = cmor_z, zfactor_name = "ps", - axis_ids = axis_ids, #[cmor_time, cmor_lat, cmor_lon], + axis_ids = axis_ids, #[cmor_time, cmor_y, cmor_x], units = "Pa" ) save_ps = True - print(f' DONE assigning cmor_lev') + print(f' DONE assigning cmor_z') axes = [] @@ -468,17 +475,17 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) appending cmor_time to axes list...') axes.append(cmor_time) print(f' axes now = {axes}') - if cmor_lev is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lev to axes list...') - axes.append(cmor_lev) + if cmor_z is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_z to axes list...') + axes.append(cmor_z) print(f' axes now = {axes}') - if cmor_lat is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lat to axes list...') - axes.append(cmor_lat) + if cmor_y is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_y to axes list...') + axes.append(cmor_y) print(f' axes now = {axes}') - if cmor_lon is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_lon to axes list...') - axes.append(cmor_lon) + if cmor_x is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_x to axes list...') + axes.append(cmor_x) print(f' axes now = {axes}') From dfe9ccb6acad3a48d9da2cc8ba294e31dc03ee2c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 9 Jan 2025 17:59:07 -0500 Subject: [PATCH 32/45] pretty sure i got it! --- fre/cmor/cmor_mixer.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index ebddc4df..960e91aa 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -453,6 +453,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') axis_ids.append(cmor_time) print(f' axis_ids now = {axis_ids}') + # might there need to be a conditional check for tripolar ocean data here as well? TODO if cmor_y is not None: print(f'(rewrite_netcdf_file_var) appending cmor_y to axis_ids list...') axis_ids.append(cmor_y) @@ -475,18 +476,23 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f'(rewrite_netcdf_file_var) appending cmor_time to axes list...') axes.append(cmor_time) print(f' axes now = {axes}') + if cmor_z is not None: print(f'(rewrite_netcdf_file_var) appending cmor_z to axes list...') axes.append(cmor_z) print(f' axes now = {axes}') - if cmor_y is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_y to axes list...') - axes.append(cmor_y) - print(f' axes now = {axes}') - if cmor_x is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_x to axes list...') - axes.append(cmor_x) - print(f' axes now = {axes}') + + if process_tripolar_data: + axes.append(cmor_grid) + else: + if cmor_y is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_y to axes list...') + axes.append(cmor_y) + print(f' axes now = {axes}') + if cmor_x is not None: + print(f'(rewrite_netcdf_file_var) appending cmor_x to axes list...') + axes.append(cmor_x) + print(f' axes now = {axes}') # read positive/units attribute and create cmor_var From 6d788b1f787e29912998650285fae128943e3761 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 9 Jan 2025 18:35:29 -0500 Subject: [PATCH 33/45] remove scratchwork script. --- run_test_file_cases.py | 4 +- ...hwork_rewrite_sos_ocean_monthly_gn_file.py | 154 ------------------ 2 files changed, 2 insertions(+), 156 deletions(-) delete mode 100644 scratchwork_rewrite_sos_ocean_monthly_gn_file.py diff --git a/run_test_file_cases.py b/run_test_file_cases.py index f7586156..f74e8dcf 100644 --- a/run_test_file_cases.py +++ b/run_test_file_cases.py @@ -89,12 +89,12 @@ def run_cmor_RUN(filename, table, opt_var_name): print(f'exception caught: exc=\n{exc}') some_return=-1 - print_the_outcome(some_return,'ocean_monthly_gn / sos') + print_the_outcome(some_return,'ocean_monthly_gn / sos') #sys.exit() -if False: +if True: # 1) SUCCEEDs # land, Lmon, gr1 testfile_land_gr1_Lmon = \ diff --git a/scratchwork_rewrite_sos_ocean_monthly_gn_file.py b/scratchwork_rewrite_sos_ocean_monthly_gn_file.py deleted file mode 100644 index 04f7167f..00000000 --- a/scratchwork_rewrite_sos_ocean_monthly_gn_file.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -#import sys -#import os -from pathlib import Path - -import numpy -#from numpy.dtypes import StringDType -from netCDF4 import Dataset #, stringtochar - - - -# open netcdf file in append mode? write mode? read? -# -# -sos_gn_fin=Dataset('./tmp/.nc', mode='r') -sos_gn_fin_ncattrs=sos_gn_fin.__dict__ #dictionary - -# the target data of interest -sos_gn_var_data = sos_gn_fin.variables['sos'][:] -sos_gn_var_atts = sos_gn_fin.variables['sos'].__dict__ - -# coordinate variables, their _bnds, and their identically named dimensions -# coordinate variable == a variable with the same name as a dimension. -# pitfall: an "axis" in netcdf is not analagous to a dimension, overloaded term -bnds_coord_data = sos_gn_fin.variables['bnds'][:] -bnds_coord_atts = sos_gn_fin.variables['bnds'].__dict__ -bnds_coord_dims = sos_gn_fin.dimensions['bnds'].size - -time_coord_data = sos_gn_fin.variables['time'][:] -time_coord_atts = sos_gn_fin.variables['time'].__dict__ -time_coord_bnds = sos_gn_fin.variables['time_bnds'][:] -time_coord_bnds_atts = sos_gn_fin.variables['time_bnds'].__dict__ -#time_coord_dims = sos_gn_fin.dimensions['time'].size - -lat_coord_data = sos_gn_fin.variables['lat'][:] -lat_coord_atts = sos_gn_fin.variables['lat'].__dict__ -lat_coord_bnds = sos_gn_fin.variables['lat_bnds'][:] -lat_coord_bnds_atts = sos_gn_fin.variables['lat_bnds'].__dict__ -lat_coord_dims = sos_gn_fin.dimensions['lat'].size - -lon_coord_data = sos_gn_fin.variables['lon'][:] -lon_coord_atts = sos_gn_fin.variables['lon'].__dict__ -lon_coord_bnds = sos_gn_fin.variables['lon_bnds'][:] -lon_coord_bnds_atts = sos_gn_fin.variables['lon_bnds'].__dict__ -lon_coord_dims = sos_gn_fin.dimensions['lon'].size - -''' - we're going to essentially re-create the most important parts of the file and see if i can't make it sort of work - recall, a netCDF4 file is, basically, 4 sets of things - attributes, i.e effectively global metadata - groups, i.e. a heirarchy with nesting a lot like directories (older netcdf files only have a root group) - dimensions, i.e. a set of named-integers to define the number of divisions on an axis - variables, i.e. arrays representing data with shapes described by the dimensions in the file -''' - -# open the output file -sos_gn_fout=Dataset('./alt_sos_gn_input/PLAY_.nc',mode='w') -sos_gn_fout.setncatts(sos_gn_fin_ncattrs) - -''' - from netCDF4 python API doc, for easy referencing - createDimension(self, - dimname, size=None)... None will imply unlimited -''' -sos_gn_fout.createDimension( 'time', - None ) #time_coord_dims -sos_gn_fout.createDimension( 'bnds', - bnds_coord_dims ) - -sos_gn_fout.createDimension( 'lat', - lat_coord_dims ) -sos_gn_fout.createDimension( 'lon', - lon_coord_dims ) - - - - -''' - from netCDF4 python API doc, for easy referencing - def createVariable(self, - varname, datatype, dimensions=(), + lots others ) -''' -# easy variables first. -# bnds -sos_gn_fout.createVariable( 'bnds', sos_gn_fin.variables['bnds'].dtype, - dimensions = ( sos_gn_fout.dimensions['bnds'] ) ) -sos_gn_fout.variables['bnds'][:] = bnds_coord_data -sos_gn_fout.variables['bnds'].setncatts( bnds_coord_atts ) - -# time -sos_gn_fout.createVariable( 'time', sos_gn_fin.variables['time'].dtype, - dimensions = ( sos_gn_fout.dimensions['time'] ) ) -sos_gn_fout.variables['time'][:] = time_coord_data -sos_gn_fout.variables['time'].setncatts( time_coord_atts ) - -# time_bnds -sos_gn_fout.createVariable( 'time_bnds', sos_gn_fin.variables['time_bnds'].dtype, - fill_value = sos_gn_fin.variables['time_bnds']._FillValue, # necessary bc of unlimited + extra limited dim shape? - dimensions = ( sos_gn_fout.dimensions['time'], - sos_gn_fout.dimensions['bnds'] ) ) -sos_gn_fout.variables['time_bnds'][:] = time_coord_bnds -for att in time_coord_bnds_atts: #sos_gn_fout.variables['time_bnds'].setncatts( time_coord_bnds_atts ) - if att != '_FillValue': - sos_gn_fout.variables['time_bnds'].setncattr( att, time_coord_bnds_atts[att] ) - -# lat -sos_gn_fout.createVariable( 'lat', sos_gn_fin.variables['lat'].dtype, - dimensions = ( sos_gn_fout.dimensions['lat'] ) ) -sos_gn_fout.variables['lat'][:] = lat_coord_data -sos_gn_fout.variables['lat'].setncatts( lat_coord_atts ) - -# lat_bnds -sos_gn_fout.createVariable( 'lat_bnds', sos_gn_fin.variables['lat_bnds'].dtype, - dimensions = ( sos_gn_fout.dimensions['lat'], - sos_gn_fout.dimensions['bnds'] ) ) -sos_gn_fout.variables['lat_bnds'][:] = lat_coord_bnds -sos_gn_fout.variables['lat_bnds'].setncatts( lat_coord_bnds_atts ) - -# lon -sos_gn_fout.createVariable( 'lon', sos_gn_fin.variables['lon'].dtype, - dimensions = ( sos_gn_fout.dimensions['lon'] ) ) -sos_gn_fout.variables['lon'][:] = lon_coord_data -sos_gn_fout.variables['lon'].setncatts( lon_coord_atts ) - -# lon_bnds -sos_gn_fout.createVariable( 'lon_bnds', sos_gn_fin.variables['lon_bnds'].dtype, - dimensions = ( sos_gn_fout.dimensions['lon'], - sos_gn_fout.dimensions['bnds'] ) ) -sos_gn_fout.variables['lon_bnds'][:] = lon_coord_bnds -sos_gn_fout.variables['lon_bnds'].setncatts( lon_coord_bnds_atts ) - -# data time!!! -sos_gn_fout.createVariable( 'sos', sos_gn_fin.variables['sos'].dtype, - fill_value = sos_gn_fin.variables['sos']._FillValue, - dimensions = ( sos_gn_fout.dimensions['time'], - None, #TODO SHOULD NOT BE NONE!!!! - sos_gn_fout.dimensions['lat'], - sos_gn_fout.dimensions['lon'] ) ) -sos_gn_fout.variables['sos'][:] = sos_gn_var_data -for att in sos_gn_var_atts: - if att not in ["time_avg_info", "_FillValue"]: - sos_gn_fout.variables['sos'].setncattr(att, sos_gn_var_atts[att] ) - -sos_gn_fout.close() - -## test that the two are equivalent "quickly"... -#unmsk_sos_gn_var_data=sos_gn_var_data[~sos_gn_var_data.mask] -#unmsk_sos_gn_var_out=sos_gn_var_out[~sos_gn_var_out.mask] -#for i in range(0, len( unmsk_sos_gn_var_data ) ): -# if i%100 == 0: -# print(f'i = {i}') -# diff = unmsk_sos_gn_var_data[i] - unmsk_sos_gn_var_out[i] -# if diff > 0.: -# print(f'diff = \n {diff}') From baf61dcc9bd2d80048f82f9944000dc0497fb6bb Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 10:40:03 -0500 Subject: [PATCH 34/45] lets just make run_test_file_cases a bunch of xfail pytests with a *ppan-only* breadcrumb for now. --- .../cmor/tests/test_cmor_run_subtool_ppan_only.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename run_test_file_cases.py => fre/cmor/tests/test_cmor_run_subtool_ppan_only.py (100%) diff --git a/run_test_file_cases.py b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py similarity index 100% rename from run_test_file_cases.py rename to fre/cmor/tests/test_cmor_run_subtool_ppan_only.py From da8b4d3aa49a9e1de3186bd255c8931f2c07144c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 11:35:45 -0500 Subject: [PATCH 35/45] ok got a format and a conditional pytest.xfail approach i am OK with. push to make sure this xfails in the pipeline appropriately --- fre/cmor/cmor_helpers.py | 1 + fre/cmor/tests/test_cmor_run_subtool.py | 6 +- .../tests/test_cmor_run_subtool_ppan_only.py | 379 +++++++++--------- 3 files changed, 196 insertions(+), 190 deletions(-) diff --git a/fre/cmor/cmor_helpers.py b/fre/cmor/cmor_helpers.py index c4707c6d..bab9899e 100644 --- a/fre/cmor/cmor_helpers.py +++ b/fre/cmor/cmor_helpers.py @@ -83,6 +83,7 @@ def get_var_filenames(indir, var_filenames = None, local_var = None): var_filenames = [] filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc' print(f'(get_var_filenames) filename_pattern = {filename_pattern}\n') + print(f'(get_var_filenames) indir = {indir}\n') var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}') #print(f'(get_var_filenames) var_filenames_all = {var_filenames_all}') for var_file in var_filenames_all: diff --git a/fre/cmor/tests/test_cmor_run_subtool.py b/fre/cmor/tests/test_cmor_run_subtool.py index a896f250..59cf1ac0 100644 --- a/fre/cmor/tests/test_cmor_run_subtool.py +++ b/fre/cmor/tests/test_cmor_run_subtool.py @@ -14,14 +14,14 @@ ROOTDIR = 'fre/tests/test_files' # setup- cmip/cmor variable table(s) -CLONE_REPO_PATH = \ +CMIP6_TABLE_REPO_PATH = \ f'{ROOTDIR}/cmip6-cmor-tables' TABLE_CONFIG = \ - f'{CLONE_REPO_PATH}/Tables/CMIP6_Omon.json' + f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_Omon.json' def test_setup_cmor_cmip_table_repo(): ''' setup routine, make sure the recursively cloned tables exist ''' - assert all( [ Path(CLONE_REPO_PATH).exists(), + assert all( [ Path(CMIP6_TABLE_REPO_PATH).exists(), Path(TABLE_CONFIG).exists() ] ) diff --git a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py index f74e8dcf..b7be5cc2 100644 --- a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py +++ b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py @@ -1,16 +1,19 @@ -#!/usr/bin/env python ''' -this is a quick and dirty script. -it will not be maintained. it will not be supported. -it is for a very context-dependent set of tests for a very specific point in time. +expanded set of tests for fre cmor run +focus on ''' -import sys +from datetime import date import os from pathlib import Path +import sys +import shutil + +import pytest from fre.cmor import cmor_run_subtool +# helper functions, not tests def print_cwd(): print(f'os.getcwd() = {os.getcwd()}') print(f'\n\n\n\n') @@ -30,195 +33,197 @@ def print_the_outcome(some_return,case_str): ROOTDIR='fre/tests/test_files' CMORBITE_VARLIST=f'{ROOTDIR}/CMORbite_var_list.json' +# cmip6 variable table(s) +CMIP6_TABLE_REPO_PATH = \ + f'{ROOTDIR}/cmip6-cmor-tables' + +# outputs +OUTDIR = f'{ROOTDIR}/outdir' +TMPDIR = f'{OUTDIR}/tmp' +# determined by cmor_run_subtool +YYYYMMDD = date.today().strftime('%Y%m%d') +CMOR_CREATES_DIR_BASE = \ + 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1'#Omon/sos/gn' +#FULL_OUTPUTDIR = \ +# f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}" + # this file exists basically for users to specify their own information to append to the netcdf file # i.e., it fills in FOO/BAR/BAZ style values, and what they are currently is totally irrelevant EXP_CONFIG_DEFAULT=f'{ROOTDIR}/CMOR_input_example.json' # this likely is not sufficient -def run_cmor_RUN(filename, table, opt_var_name): - func_debug1 = False - if func_debug1: - print('cmor_run_subtool(\n' - f' indir = \"{str(Path(filename).parent)}\",\n' - f' json_var_list = \"{CMORBITE_VARLIST}\",\n' - f' json_table_config = \"{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_{table}.json\",\n' - f' json_exp_config = \"{EXP_CONFIG_DEFAULT}\",\n' - f' outdir = \"{os.getcwd()}\",\n' - f' opt_var_name = \"{opt_var_name}\"\n' - ')\n' - ) - func_debug2 = True - if func_debug2: - print('fre cmor run ' - f'-d {str(Path(filename).parent)} ' - f'-l {CMORBITE_VARLIST} ' - f'-r {ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_{table}.json ' - f'-p {EXP_CONFIG_DEFAULT} ' - f'-o {os.getcwd()} ' - f'-v {opt_var_name} ' - ) - FOO_return = cmor_run_subtool( - indir = str(Path(filename).parent), - json_var_list = CMORBITE_VARLIST, - json_table_config = f'{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_{table}.json', - json_exp_config = EXP_CONFIG_DEFAULT, - outdir = os.getcwd(), # dont keep it this way... - opt_var_name = opt_var_name - ) - return FOO_return - - - -if True: - #### THIS CASE MAY WORK if i rewrite the ocean file correctly, effectively appending the lat/lon data from a statics file. - #### for this case, that file is: - #### '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - #### 'pp/ocean_monthly/' + \ - #### 'ocean_monthly.static.nc' - #### and that data is stored under "geolon" and "geolat" consuming dims "x" and "y". - # 6) FAIL - # ocean, Omon / sos - # Result - error, it wants lat/lon, but only xh, yh coordinates are available - testfile_ocean_monthly_gn = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly/ts/monthly/5yr/' + \ - 'ocean_monthly.002101-002512.sos.nc' - try: - some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'ocean_monthly_gn / sos') - - #sys.exit() - - -if True: - # 1) SUCCEEDs - # land, Lmon, gr1 - testfile_land_gr1_Lmon = \ + +def test_case_land_Lmon_gr1(): + + # clean up from previous tests + if Path(f'{OUTDIR}/CMIP6').exists(): + shutil.rmtree(f'{OUTDIR}/CMIP6') + + # define inputs to the cmor run tool + testfile = \ '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ 'pp/land/ts/monthly/5yr/' + \ 'land.005101-005512.lai.nc' + indir = Path(testfile).parent + table = 'Lmon' + table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' + opt_var_name = 'lai' + + # if we can't find the input test file, do an xfail. most likely, you're not at PPAN. + if not Path(testfile).exists(): + pytest.xfail('land, Lmon, gr1 - SUCCEEDs on PP/AN at GFDL only! OR testfile does not exist!') + + # execute the test try: - some_return = run_cmor_RUN(testfile_land_gr1_Lmon, 'Lmon', opt_var_name = 'lai') - except: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'land_gr1_Lmon / lai') - - - # 2) SUCCEEDs - # atmos, Amon / cl - testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.196001-196412.cl.nc' - try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') - - - # 3) SUCCEEDs - # atmos, Amon / mc - testfile_atmos_level_cmip_gr1_Amon_fullL = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ - 'atmos_level_cmip.195501-195912.mc.nc' - try: - some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') - - - # 4) SUCCEEDs (no longitude coordinate case) - # atmos, AERmonZ / ta - # just like #1, but lack longitude - testfile_atmos_gr1_AERmonZ_nolons = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ - 'atmos_plev39_cmip.201001-201412.ta.nc' - try: - some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') - - - # 5) SUCCEEDs - # ocean, Omon / sos - testfile_ocean_monthly_1x1deg_gr = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_1x1deg.190001-190412.sos.nc' - try: - some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') - - - # 7) SUCCEEDs - # ocean, Omon / so - testfile_ocean_monthly_z_1x1deg_gr = \ - '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ - 'ocean_monthly_z_1x1deg.000101-000512.so.nc' - try: - some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') - - - # 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) - # atmos, Amon / ch4global - testfile_atmos_scalar_gn_Amon_nolon_nolat = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/atmos_scalar/ts/monthly/5yr/' + \ - 'atmos_scalar.197001-197412.ch4global.nc' - try: - some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') - except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') - - - - # 9) SUCCEEDs (needs coordinate variable axis with character string values) - # land, Emon / gppLut - testfile_LUmip_refined_gr1_Emon_landusedim = \ - '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ - 'pp/LUmip_refined/ts/monthly/5yr/' + \ - 'LUmip_refined.185001-185412.gppLut.nc' - try: - some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') + cmor_run_subtool( + indir = indir, + json_var_list = CMORBITE_VARLIST, + json_table_config = table_file, + json_exp_config = EXP_CONFIG_DEFAULT, + outdir = OUTDIR, + opt_var_name = opt_var_name + ) + some_return = 0 except Exception as exc: - print(f'exception caught: exc=\n{exc}') - some_return=-1 - - print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') - - - - - - - + raise Exception(f'exception caught: exc=\n{exc}') from exc + + # outputs that should be created + cmor_output_dir = f'{OUTDIR}/{CMOR_CREATES_DIR_BASE}/{table}/{opt_var_name}/gn/v{YYYYMMDD}' + cmor_output_file = f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_000101-000601.nc' + + # success criteria + assert all( [ some_return == 0, + Path(cmor_output_dir).exists(), + Path(cmor_output_file).exists() ] ) + + +# +## 2) SUCCEEDs +## atmos, Amon / cl +#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.196001-196412.cl.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') +# +# +## 3) SUCCEEDs +## atmos, Amon / mc +#testfile_atmos_level_cmip_gr1_Amon_fullL = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ +# 'atmos_level_cmip.195501-195912.mc.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') +# +# +## 4) SUCCEEDs (no longitude coordinate case) +## atmos, AERmonZ / ta +## just like #1, but lack longitude +#testfile_atmos_gr1_AERmonZ_nolons = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ +# 'atmos_plev39_cmip.201001-201412.ta.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') +# +# +## 5) SUCCEEDs +## ocean, Omon / sos, gr +#testfile_ocean_monthly_1x1deg_gr = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_1x1deg.190001-190412.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') +# +# +## 6) SUCCEEDs +## ocean, Omon / sos, gn +#testfile_ocean_monthly_gn = \ +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly/ts/monthly/5yr/' + \ +# 'ocean_monthly.002101-002512.sos.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +# print_the_outcome(some_return,'ocean_monthly_gn / sos') +# +# +# +## 7) SUCCEEDs +## ocean, Omon / so +#testfile_ocean_monthly_z_1x1deg_gr = \ +# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ +# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ +# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' +#try: +# some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') +# +# +## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) +## atmos, Amon / ch4global +#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/atmos_scalar/ts/monthly/5yr/' + \ +# 'atmos_scalar.197001-197412.ch4global.nc' +#try: +# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') +# +# +# +## 9) SUCCEEDs (needs coordinate variable axis with character string values) +## land, Emon / gppLut +#testfile_LUmip_refined_gr1_Emon_landusedim = \ +# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ +# 'pp/LUmip_refined/ts/monthly/5yr/' + \ +# 'LUmip_refined.185001-185412.gppLut.nc' +#try: +# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') +#except Exception as exc: +# print(f'exception caught: exc=\n{exc}') +# some_return=-1 +# +#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') +# +# +# +# +# +# +# +# From 5d7f77dcc15cf3f55e1bc361eba2b846394f84f7 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 13:06:33 -0500 Subject: [PATCH 36/45] cleaner tests, all should xfail appropriately in the CI/CD --- .../tests/test_cmor_run_subtool_ppan_only.py | 290 ++++++++---------- 1 file changed, 125 insertions(+), 165 deletions(-) diff --git a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py index b7be5cc2..9274068e 100644 --- a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py +++ b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py @@ -1,6 +1,6 @@ ''' expanded set of tests for fre cmor run -focus on +focus on ''' from datetime import date @@ -9,26 +9,12 @@ import sys import shutil +import glob + import pytest from fre.cmor import cmor_run_subtool -# helper functions, not tests -def print_cwd(): - print(f'os.getcwd() = {os.getcwd()}') - print(f'\n\n\n\n') - -def print_the_outcome(some_return,case_str): - print('-----------------------------------------------------------------------------------------------------------------') - if some_return != 0: - print(f'{case_str} case failed [[[ FAIL -_- ]]]: some_return={some_return}') - else: - print(f'{case_str} case probably OK [[[ PROB-OK ^-^ ]]]: some_return={some_return}') - print('-----------------------------------------------------------------------------------------------------------------') - print(f'\n\n\n\n\n\n\n\n\n\n') - print_cwd() - #assert some_return == 0 - # global consts for these tests, with no/trivial impact on the results ROOTDIR='fre/tests/test_files' CMORBITE_VARLIST=f'{ROOTDIR}/CMORbite_var_list.json' @@ -40,38 +26,38 @@ def print_the_outcome(some_return,case_str): # outputs OUTDIR = f'{ROOTDIR}/outdir' TMPDIR = f'{OUTDIR}/tmp' + # determined by cmor_run_subtool YYYYMMDD = date.today().strftime('%Y%m%d') CMOR_CREATES_DIR_BASE = \ - 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1'#Omon/sos/gn' -#FULL_OUTPUTDIR = \ -# f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}" + 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1' # this file exists basically for users to specify their own information to append to the netcdf file # i.e., it fills in FOO/BAR/BAZ style values, and what they are currently is totally irrelevant -EXP_CONFIG_DEFAULT=f'{ROOTDIR}/CMOR_input_example.json' # this likely is not sufficient +EXP_CONFIG_DEFAULT=f'{ROOTDIR}/CMOR_input_example.json' # this likely is not sufficient +CLEANUP_AFTER_EVERY_TEST = False - -def test_case_land_Lmon_gr1(): - +def _cleanup(): # clean up from previous tests if Path(f'{OUTDIR}/CMIP6').exists(): shutil.rmtree(f'{OUTDIR}/CMIP6') + assert not Path(f'{OUTDIR}/CMIP6').exists() + +def _case_function(testfile_dir, table, opt_var_name, grid): # define inputs to the cmor run tool - testfile = \ - '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ - 'pp/land/ts/monthly/5yr/' + \ - 'land.005101-005512.lai.nc' - indir = Path(testfile).parent - table = 'Lmon' + #testfile = testfile + indir = testfile_dir#Path(testfile).parent + table = table table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' - opt_var_name = 'lai' + opt_var_name = opt_var_name + grid = grid # if we can't find the input test file, do an xfail. most likely, you're not at PPAN. - if not Path(testfile).exists(): - pytest.xfail('land, Lmon, gr1 - SUCCEEDs on PP/AN at GFDL only! OR testfile does not exist!') + if not Path(testfile_dir).exists(): + pytest.xfail(f'{opt_var_name}, {Path(table_file).name}, {grid} ' + 'SUCCEEDs on PP/AN at GFDL only! OR testfile_dir does not exist!') # execute the test try: @@ -89,141 +75,115 @@ def test_case_land_Lmon_gr1(): # outputs that should be created cmor_output_dir = f'{OUTDIR}/{CMOR_CREATES_DIR_BASE}/{table}/{opt_var_name}/gn/v{YYYYMMDD}' - cmor_output_file = f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_000101-000601.nc' + cmor_output_file = glob.glob(f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_??????-??????.nc')[0] # success criteria assert all( [ some_return == 0, Path(cmor_output_dir).exists(), Path(cmor_output_file).exists() ] ) - -# -## 2) SUCCEEDs -## atmos, Amon / cl -#testfile_atmos_level_cmip_gr1_Amon_complex_vert = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.196001-196412.cl.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_complex_vert, 'Amon', opt_var_name = 'cl') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_complex_vert / cl') -# -# -## 3) SUCCEEDs -## atmos, Amon / mc -#testfile_atmos_level_cmip_gr1_Amon_fullL = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_level_cmip/ts/monthly/5yr/' + \ -# 'atmos_level_cmip.195501-195912.mc.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_level_cmip_gr1_Amon_fullL, 'Amon', opt_var_name = 'mc') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'atmos_level_cmip_gr1_Amon_fullL / mc') -# -# -## 4) SUCCEEDs (no longitude coordinate case) -## atmos, AERmonZ / ta -## just like #1, but lack longitude -#testfile_atmos_gr1_AERmonZ_nolons = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + \ -# 'atmos_plev39_cmip.201001-201412.ta.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_gr1_AERmonZ_nolons, 'AERmonZ', opt_var_name = 'ta') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'atmos_gr1_AERmonZ_nolons / ta') -# -# -## 5) SUCCEEDs -## ocean, Omon / sos, gr -#testfile_ocean_monthly_1x1deg_gr = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_1x1deg.190001-190412.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_1x1deg_gr, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'ocean_monthly_1x1deg_gr / sos') -# -# -## 6) SUCCEEDs -## ocean, Omon / sos, gn -#testfile_ocean_monthly_gn = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly/ts/monthly/5yr/' + \ -# 'ocean_monthly.002101-002512.sos.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_gn, 'Omon', opt_var_name = 'sos') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -# print_the_outcome(some_return,'ocean_monthly_gn / sos') -# -# -# -## 7) SUCCEEDs -## ocean, Omon / so -#testfile_ocean_monthly_z_1x1deg_gr = \ -# '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ -# 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + \ -# 'ocean_monthly_z_1x1deg.000101-000512.so.nc' -#try: -# some_return = run_cmor_RUN(testfile_ocean_monthly_z_1x1deg_gr, 'Omon', opt_var_name = 'so') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'ocean_monthly_z_1x1deg_gr / so') -# -# -## 8) SUCCEEDs (no latitude, nor longitude, nor vertical coordinates cases) -## atmos, Amon / ch4global -#testfile_atmos_scalar_gn_Amon_nolon_nolat = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/atmos_scalar/ts/monthly/5yr/' + \ -# 'atmos_scalar.197001-197412.ch4global.nc' -#try: -# some_return = run_cmor_RUN(testfile_atmos_scalar_gn_Amon_nolon_nolat, 'Amon', opt_var_name = 'ch4global') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'atmos_scalar_gn_Amon_nolon_nolat / ch4global') -# -# -# -## 9) SUCCEEDs (needs coordinate variable axis with character string values) -## land, Emon / gppLut -#testfile_LUmip_refined_gr1_Emon_landusedim = \ -# '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ -# 'pp/LUmip_refined/ts/monthly/5yr/' + \ -# 'LUmip_refined.185001-185412.gppLut.nc' -#try: -# some_return = run_cmor_RUN(testfile_LUmip_refined_gr1_Emon_landusedim, 'Emon', opt_var_name = 'gppLut') -#except Exception as exc: -# print(f'exception caught: exc=\n{exc}') -# some_return=-1 -# -#print_the_outcome(some_return,'LUmip_refined_gr1_Emon_landusedim / gppLut') -# -# -# -# -# -# -# -# + if CLEANUP_AFTER_EVERY_TEST: + _cleanup() + +#### test cases +def test_cleanup(): + _cleanup() + + +def test_case_Lmon_lai_gr1(): + testfile_dir = \ + '/archive/Eric.Stofferahn/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/land/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Lmon', + opt_var_name = 'lai', + grid = 'gr1' + ) + + +def test_case_Amon_cl_gr1(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Amon', + opt_var_name = 'cl', + grid = 'gr1' + ) + + +def test_case_Amon_mc_gr1(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_level_cmip/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Amon', + opt_var_name = 'mc', + grid = 'gr1' + ) + + +def test_case_AERmonZ_ta_gr1(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_plev39_cmip/ts/monthly/5yr/zonavg/' + _case_function( testfile_dir = testfile_dir, + table = 'AERmonZ', + opt_var_name = 'ta', + grid = 'gr1' + ) + + +def test_case_Omon_sos_gr(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/ocean_monthly_1x1deg/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Omon', + opt_var_name = 'sos', + grid = 'gr' + ) + + +def test_case_Omon_sos_gn(): + testfile_dir = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Omon', + opt_var_name = 'sos', + grid = 'gn' + ) + + +def test_case_Omon_so_gr(): + testfile_dir = \ + '/archive/ejs/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/' + \ + 'pp/ocean_monthly_z_1x1deg/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Omon', + opt_var_name = 'so', + grid = 'gr' + ) + + +def test_case_Amon_ch4global_gn(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/atmos_scalar/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Amon', + opt_var_name = 'ch4global', + grid = 'gn' + ) + + +def test_case_Emon_gppLut_gr1(): + testfile_dir = \ + '/arch0/cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/' + \ + 'pp/LUmip_refined/ts/monthly/5yr/' + _case_function( testfile_dir = testfile_dir, + table = 'Emon', + opt_var_name = 'gppLut', + grid = 'gr1' + ) From 1c987dabe9d652941fce2f5a431f637c6d9e3100 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 13:23:48 -0500 Subject: [PATCH 37/45] pylint things. rename cmor_lister to cmor_finder to avoid clash with builtin. make max-line-length in pylint calls 120. --- .github/workflows/create_test_conda_env.yml | 2 +- fre/analysis/freanalysis.py | 2 ++ fre/check/frecheckexample.py | 2 +- fre/cmor/__init__.py | 2 +- fre/cmor/{cmor_lister.py => cmor_finder.py} | 24 +++++++++---------- fre/cmor/frecmor.py | 21 ++++++++-------- .../tests/test_cmor_run_subtool_ppan_only.py | 6 ++--- fre/list/frelistexample.py | 2 +- fre/run/frerunexample.py | 2 +- fre/test/fretestexample.py | 2 +- meta.yaml | 2 +- 11 files changed, 34 insertions(+), 33 deletions(-) rename fre/cmor/{cmor_lister.py => cmor_finder.py} (84%) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 18ff20f0..cfaebfb1 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -48,7 +48,7 @@ jobs: - name: Run pylint in fre-cli environment run: | # run pylint, ignored modules avoid warnings arising from code internal to those modules - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." + pylint --max-line-length 120 --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." - name: Install Sphinx and Build Documentation run: | diff --git a/fre/analysis/freanalysis.py b/fre/analysis/freanalysis.py index 3a85ea06..3fb4a608 100644 --- a/fre/analysis/freanalysis.py +++ b/fre/analysis/freanalysis.py @@ -1,3 +1,5 @@ +''' fre analysis ''' + # a third party package import click diff --git a/fre/check/frecheckexample.py b/fre/check/frecheckexample.py index 8d4d7536..ea24694d 100644 --- a/fre/check/frecheckexample.py +++ b/fre/check/frecheckexample.py @@ -9,7 +9,7 @@ def check_test_function(uppercase=None): statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() - click.echo(statement) + print(statement) if __name__ == '__main__': check_test_function() diff --git a/fre/cmor/__init__.py b/fre/cmor/__init__.py index eb84d10a..4bb50201 100644 --- a/fre/cmor/__init__.py +++ b/fre/cmor/__init__.py @@ -1,3 +1,3 @@ ''' for fre.cmor imports ''' from .cmor_mixer import cmor_run_subtool -from .cmor_lister import cmor_list_subtool +from .cmor_finder import cmor_find_subtool diff --git a/fre/cmor/cmor_lister.py b/fre/cmor/cmor_finder.py similarity index 84% rename from fre/cmor/cmor_lister.py rename to fre/cmor/cmor_finder.py index 1c71d42c..c0f3108b 100644 --- a/fre/cmor/cmor_lister.py +++ b/fre/cmor/cmor_finder.py @@ -1,4 +1,4 @@ -''' fre cmor list +''' fre cmor find because ian got tired of typing things like the following in bash... varname=sos; \ @@ -29,12 +29,12 @@ def print_var_content( table_config_file = None, var_name = None): try: var_content = proj_table_vars["variable_entry"].get(var_name) except: - #print(f'(cmor_list_subtool) WARNING no "variable_entry" key. for {json_table_config}.' + #print(f'(cmor_find_subtool) WARNING no "variable_entry" key. for {json_table_config}.' # ' not the right json file probably. moving on!') return if var_content is None: - #print(f'(cmor_list_subtool) variable {var_name} not found in {Path(json_table_config).name}, moving on!') + #print(f'(cmor_find_subtool) variable {var_name} not found in {Path(json_table_config).name}, moving on!') return table_name = None @@ -61,20 +61,20 @@ def print_var_content( table_config_file = None, var_name = None): return -def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): +def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_var_name = None): ''' finds tables in the CMIP json config directory containing variable data of interest. prints it out to screen, intended largely as a helper tool for cli users. ''' if not Path(json_table_config_dir).exists(): - raise OSError(f'(cmor_list_subtool) ERROR directory {json_table_config_dir} does not exist! exit.') + raise OSError(f'(cmor_find_subtool) ERROR directory {json_table_config_dir} does not exist! exit.') - print(f'(cmor_list_subtool) attempting to find and open files in dir: \n {json_table_config_dir} ') + print(f'(cmor_find_subtool) attempting to find and open files in dir: \n {json_table_config_dir} ') json_table_configs=glob.glob(f'{json_table_config_dir}/CMIP6_*.json') if json_table_configs is None: raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.') else: - print(f'(cmor_list_subtool) found content in json_table_config_dir')#: {json_table_configs}') + print(f'(cmor_find_subtool) found content in json_table_config_dir')#: {json_table_configs}') var_list = None if json_var_list is not None: @@ -82,20 +82,20 @@ def cmor_list_subtool( json_var_list = None, json_table_config_dir = None, opt_v var_list=json.load(var_list_file) if opt_var_name is None and var_list is None: - raise ValueError(f'(cmor_list_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') + raise ValueError(f'(cmor_find_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') if opt_var_name is not None: - print(f'(cmor_list_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') + print(f'(cmor_find_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') for json_table_config in json_table_configs: - #print(f'(cmor_list_subtool) attempting to open {json_table_config}') + #print(f'(cmor_find_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: print_var_content(table_config_file, opt_var_name) elif var_list is not None: - print(f'(cmor_list_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') + print(f'(cmor_find_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') for var in var_list: for json_table_config in json_table_configs: - #print(f'(cmor_list_subtool) attempting to open {json_table_config}') + #print(f'(cmor_find_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: #print(f' var = {var}, var_list[{var}]={var_list[var]}') print_var_content(table_config_file, str(var_list[var])) diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py index 8e42e2fb..40505dcf 100644 --- a/fre/cmor/frecmor.py +++ b/fre/cmor/frecmor.py @@ -2,7 +2,7 @@ import click -from .cmor_lister import cmor_list_subtool +from .cmor_finder import cmor_find_subtool from .cmor_mixer import cmor_run_subtool OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \ @@ -49,9 +49,9 @@ def cmor_cli(): required=False) def run(indir, varlist, table_config, exp_config, outdir, opt_var_name): # pylint: disable=unused-argument - """ - Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing - """ + """ + Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing + """ cmor_run_subtool( indir = indir, json_var_list = varlist, @@ -75,16 +75,15 @@ def run(indir, varlist, table_config, exp_config, outdir, opt_var_name): type = str, help=OPT_VAR_NAME_HELP, required=False) -def list(varlist, table_config_dir, opt_var_name): +def find(varlist, table_config_dir, opt_var_name): ''' loop over json table files in config_dir and show which tables contain variables in var list/ the tool will also print what that table entry is expecting of that variable as well. if given an opt_var_name in addition to varlist, only that variable name will be printed out. - accepts 3 arguments, two of the three required. + accepts 3 arguments, two of the three required. ''' - #context.forward( _cmor_list_subtool ) - # if opt_var_name specified, forget the list. + # if opt_var_name specified, forget the list. if opt_var_name is not None: varlist=None @@ -92,13 +91,13 @@ def list(varlist, table_config_dir, opt_var_name): # logic before calling context.invoke( , *args ) if opt_var_name is None and varlist is None: raise ValueError('opt_var_name and varlist cannot both be None') - - cmor_list_subtool( + + cmor_find_subtool( json_var_list = varlist, json_table_config_dir = table_config_dir, opt_var_name = opt_var_name ) - + if __name__ == "__main__": cmor_cli() diff --git a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py index 9274068e..df5d1f52 100644 --- a/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py +++ b/fre/cmor/tests/test_cmor_run_subtool_ppan_only.py @@ -47,8 +47,7 @@ def _cleanup(): def _case_function(testfile_dir, table, opt_var_name, grid): # define inputs to the cmor run tool - #testfile = testfile - indir = testfile_dir#Path(testfile).parent + indir = testfile_dir table = table table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' opt_var_name = opt_var_name @@ -75,7 +74,8 @@ def _case_function(testfile_dir, table, opt_var_name, grid): # outputs that should be created cmor_output_dir = f'{OUTDIR}/{CMOR_CREATES_DIR_BASE}/{table}/{opt_var_name}/gn/v{YYYYMMDD}' - cmor_output_file = glob.glob(f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_??????-??????.nc')[0] + cmor_output_file = glob.glob( + f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_??????-??????.nc')[0] # success criteria assert all( [ some_return == 0, diff --git a/fre/list/frelistexample.py b/fre/list/frelistexample.py index 8423ed14..4cf73028 100644 --- a/fre/list/frelistexample.py +++ b/fre/list/frelistexample.py @@ -9,7 +9,7 @@ def list_test_function(uppercase=None): statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() - click.echo(statement) + print(statement) if __name__ == '__main__': list_test_function() diff --git a/fre/run/frerunexample.py b/fre/run/frerunexample.py index c5baa107..233d3310 100644 --- a/fre/run/frerunexample.py +++ b/fre/run/frerunexample.py @@ -9,7 +9,7 @@ def run_test_function(uppercase=None): statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() - click.echo(statement) + print(statement) if __name__ == '__main__': run_test_function() diff --git a/fre/test/fretestexample.py b/fre/test/fretestexample.py index 4daa5447..e1215f6a 100644 --- a/fre/test/fretestexample.py +++ b/fre/test/fretestexample.py @@ -9,7 +9,7 @@ def test_test_function(uppercase=None): statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() - click.echo(statement) + print(statement) if __name__ == '__main__': test_test_function() diff --git a/meta.yaml b/meta.yaml index 2c9fdcde..8ec674d0 100644 --- a/meta.yaml +++ b/meta.yaml @@ -65,7 +65,7 @@ test: - fre.test - fre.yamltools commands: - - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." + - pylint --max-line-length 120 --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." # run pytest but ignore any tests that require compilation - pytest --ignore=fre/make/tests/compilation --config-file=fre/pytest.ini --cov-report term-missing --cov-config=fre/coveragerc --cov=fre fre/ - fre --help From aa6c6125514d42f41dbb2e9a856f5689c64488f0 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 13:33:22 -0500 Subject: [PATCH 38/45] fix an oops in the fre cmor cli tests. tweak pytest.ini trivially --- fre/pytest.ini | 8 ++++---- fre/tests/test_fre_cmor_cli.py | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/fre/pytest.ini b/fre/pytest.ini index e8822023..6cc50cb0 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -1,6 +1,9 @@ [pytest] testpaths = fre/tests + fre/analysis/tests + fre/app/generate_time_averages/tests + fre/app/regrid_xy/tests fre/catalog/tests # fre/check/tests fre/cmor/tests @@ -10,8 +13,5 @@ testpaths = # fre/run/tests # fre/test/tests fre/yamltools/tests -# fre/app/tests - fre/app/generate_time_averages/tests - fre/app/regrid_xy/tests - fre/analysis/tests + diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index fa07595f..195965f4 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -152,18 +152,18 @@ def test_cli_fre_cmor_run_case2(capfd): Path(full_inputfile).exists() ] ) _out, _err = capfd.readouterr() -# fre cmor list -def test_cli_fre_cmor_list(): - ''' fre cmor list ''' - result = runner.invoke(fre.fre, args=["cmor", "list"]) +# fre cmor find +def test_cli_fre_cmor_find(): + ''' fre cmor find ''' + result = runner.invoke(fre.fre, args=["cmor", "find"]) assert result.exit_code == 2 -def test_cli_fre_cmor_list_help(): - ''' fre cmor list --help ''' - result = runner.invoke(fre.fre, args=["cmor", "list", "--help"]) +def test_cli_fre_cmor_find_help(): + ''' fre cmor find --help ''' + result = runner.invoke(fre.fre, args=["cmor", "find", "--help"]) assert result.exit_code == 0 -def test_cli_fre_cmor_list_opt_dne(): - ''' fre cmor list optionDNE ''' - result = runner.invoke(fre.fre, args=["cmor", "list", "optionDNE"]) +def test_cli_fre_cmor_find_opt_dne(): + ''' fre cmor find optionDNE ''' + result = runner.invoke(fre.fre, args=["cmor", "find", "optionDNE"]) assert result.exit_code == 2 From fd73c43e4c528e623baab44e136c16c9001e9b3c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 13:58:28 -0500 Subject: [PATCH 39/45] update workflow file to upload the coverage xml for inspection. change coveragerc to not ignore test directories- might be why the coverage results are wonky --- .github/workflows/create_test_conda_env.yml | 21 ++++++++++++++------- fre/coveragerc | 8 -------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index cfaebfb1..3bf3a621 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -13,8 +13,7 @@ jobs: submodules: 'recursive' - name: Create fre-cli environment run: | - # create environment containing all dependencies - # the env cannot be explicitly activated in github CI/CD + # create env holding all deps, the env cant be explicitly activated in CI/CD conda env create -f environment.yml --name fre-cli # sets CONDA to wherever it may be on the image @@ -24,15 +23,12 @@ jobs: echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH echo $PWD/mkmf/bin >> $GITHUB_PATH - # use *conda environment's pip* to install fre-cli - # called w/ full path to conda's python for explicitness - # called as a module (-m pip) for explicitness + # use *conda environment's pip* to install fre-cli, called w/ full path as module for explicitness $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . - name: Run pytest in fre-cli environment run: | - # add spack installed binaries to front of path so that - # conda's netcdf/hdf5 installs don't break compilation tests + # add spack installed binaries to front of path so that conda's netcdf/hdf5 installs don't break compilation tests export path_save=$PATH export PATH="/opt/views/view/bin:$PATH" @@ -41,10 +37,18 @@ jobs: # restore original path and install genbadge to generate coverage badge based on xml export PATH="$path_save" + + # install genbadge to make badge from coverage/test stats pip install genbadge genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg + - name: Archive code coverage results + uses: actions/upload-artifact@v4 + with: + name: code-coverage-report + path: coverage.xml + - name: Run pylint in fre-cli environment run: | # run pylint, ignored modules avoid warnings arising from code internal to those modules @@ -52,8 +56,11 @@ jobs: - name: Install Sphinx and Build Documentation run: | + # pip install sphinx and themes, upgrade theme pip install sphinx renku-sphinx-theme sphinx-rtd-theme pip install --upgrade sphinx-rtd-theme + + # have sphinx build the docs sphinx-apidoc --output-dir docs fre/ --separate sphinx-build docs build diff --git a/fre/coveragerc b/fre/coveragerc index 0e936b95..dd4cdab6 100644 --- a/fre/coveragerc +++ b/fre/coveragerc @@ -1,15 +1,7 @@ # https://pytest-cov.readthedocs.io/en/latest/config.html [run] omit = - fre/tests/* - fre/app/generate_time_averages/tests/* - fre/app/regrid_xy/tests/* - fre/catalog/tests/* fre/check - fre/cmor/tests/* fre/list - fre/make/tests/* - fre/pp/tests/* fre/run fre/test - fre/yamltools/tests/* From c428716908efaefb8aa06618ea84135bde70cde8 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 14:37:37 -0500 Subject: [PATCH 40/45] why no coverage badge?? smh.. --- .github/workflows/build_conda.yml | 10 +++++++++- .github/workflows/create_test_conda_env.yml | 4 ++++ fre/coveragerc | 18 +++++++++++++----- fre/pytest.ini | 8 +------- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 94acc688..ca0e562d 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -13,12 +13,20 @@ jobs: uses: actions/checkout@v4 with: submodules: 'recursive' - - name: Add mkmf to PATH + + - name: Add mkmf to GITHUB_PATH run: | + # add mkmf to GITHUB_PATH echo $PWD/mkmf/bin >> $GITHUB_PATH + - name: Run Conda to Build run: | + # append the reqd channels conda config --append channels conda-forge conda config --append channels noaa-gfdl + + # install conda-build and conda-verify conda install conda-build conda-verify + + # conda build conda build . diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 3bf3a621..704b5a84 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -40,7 +40,11 @@ jobs: # install genbadge to make badge from coverage/test stats pip install genbadge + + # genbadge coverage genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg + + # genbadge tests genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg - name: Archive code coverage results diff --git a/fre/coveragerc b/fre/coveragerc index dd4cdab6..6fbcf443 100644 --- a/fre/coveragerc +++ b/fre/coveragerc @@ -1,7 +1,15 @@ -# https://pytest-cov.readthedocs.io/en/latest/config.html [run] omit = - fre/check - fre/list - fre/run - fre/test + fre/check/* + fre/list/* + fre/run/* + fre/test/* + fre/tests/* + fre/analysis/tests/* + fre/app/generate_time_averages/tests/* + fre/app/regrid_xy/tests/* + fre/catalog/tests/* + fre/cmor/tests/* + fre/make/tests/* + fre/pp/tests/* + fre/yamltools/tests/* diff --git a/fre/pytest.ini b/fre/pytest.ini index 6cc50cb0..e6d03f0b 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -5,13 +5,7 @@ testpaths = fre/app/generate_time_averages/tests fre/app/regrid_xy/tests fre/catalog/tests -# fre/check/tests fre/cmor/tests -# fre/list/tests fre/make/tests fre/pp/tests -# fre/run/tests -# fre/test/tests - fre/yamltools/tests - - + fre/yamltools/tests \ No newline at end of file From dd21a0c847bf2b0df8cbf36f3850a2abce9dfddd Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 14:51:46 -0500 Subject: [PATCH 41/45] put coveragerc and pytest.ini back to what they were... sorry for touching them... sheesh --- fre/coveragerc | 9 --------- fre/pytest.ini | 8 ++++---- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/fre/coveragerc b/fre/coveragerc index 6fbcf443..66d6ebfe 100644 --- a/fre/coveragerc +++ b/fre/coveragerc @@ -4,12 +4,3 @@ omit = fre/list/* fre/run/* fre/test/* - fre/tests/* - fre/analysis/tests/* - fre/app/generate_time_averages/tests/* - fre/app/regrid_xy/tests/* - fre/catalog/tests/* - fre/cmor/tests/* - fre/make/tests/* - fre/pp/tests/* - fre/yamltools/tests/* diff --git a/fre/pytest.ini b/fre/pytest.ini index e6d03f0b..1842b6a8 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -1,11 +1,11 @@ [pytest] testpaths = fre/tests - fre/analysis/tests - fre/app/generate_time_averages/tests - fre/app/regrid_xy/tests fre/catalog/tests fre/cmor/tests fre/make/tests fre/pp/tests - fre/yamltools/tests \ No newline at end of file + fre/yamltools/tests + fre/analysis/tests + fre/app/generate_time_averages/tests + fre/app/regrid_xy/tests From 2d6730e7228b7dd4cc5dd5584e4fd95c2f7d9044 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 15:10:34 -0500 Subject: [PATCH 42/45] add */tests/__init__.py files back in... blegh --- fre/analysis/tests/__init__.py | 0 fre/app/generate_time_averages/tests/__init__.py | 0 fre/app/regrid_xy/tests/__init__.py | 0 fre/catalog/tests/__init__.py | 0 fre/cmor/tests/__init__.py | 0 fre/make/tests/__init__.py | 0 fre/pp/tests/__init__.py | 0 fre/tests/__init__.py | 0 fre/yamltools/tests/__init__.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 fre/analysis/tests/__init__.py create mode 100644 fre/app/generate_time_averages/tests/__init__.py create mode 100644 fre/app/regrid_xy/tests/__init__.py create mode 100644 fre/catalog/tests/__init__.py create mode 100644 fre/cmor/tests/__init__.py create mode 100644 fre/make/tests/__init__.py create mode 100644 fre/pp/tests/__init__.py create mode 100644 fre/tests/__init__.py create mode 100644 fre/yamltools/tests/__init__.py diff --git a/fre/analysis/tests/__init__.py b/fre/analysis/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/app/generate_time_averages/tests/__init__.py b/fre/app/generate_time_averages/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/app/regrid_xy/tests/__init__.py b/fre/app/regrid_xy/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/catalog/tests/__init__.py b/fre/catalog/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/cmor/tests/__init__.py b/fre/cmor/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/make/tests/__init__.py b/fre/make/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/pp/tests/__init__.py b/fre/pp/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/tests/__init__.py b/fre/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fre/yamltools/tests/__init__.py b/fre/yamltools/tests/__init__.py new file mode 100644 index 00000000..e69de29b From be4af5c7ba7ddd7a0a1a7e9f70d400249f4a6de5 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 15:17:10 -0500 Subject: [PATCH 43/45] another testing init file --- fre/make/tests/compilation/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 fre/make/tests/compilation/__init__.py diff --git a/fre/make/tests/compilation/__init__.py b/fre/make/tests/compilation/__init__.py new file mode 100644 index 00000000..e69de29b From 1923e6d15dc0f49481c79b4e13ccc3e0c7378449 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 15:56:24 -0500 Subject: [PATCH 44/45] adjust pytest calls in workflows to use coverage to call pytest as module, for appropriate data collection. adjust paths to globs in coveragerc --- .github/workflows/create_test_conda_env.yml | 2 +- fre/coveragerc | 9 +++++++++ fre/pytest.ini | 6 +++--- meta.yaml | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 704b5a84..7c67b92f 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -33,7 +33,7 @@ jobs: export PATH="/opt/views/view/bin:$PATH" # run pytest - pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ + coverage run -m pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ # restore original path and install genbadge to generate coverage badge based on xml export PATH="$path_save" diff --git a/fre/coveragerc b/fre/coveragerc index 66d6ebfe..6fbcf443 100644 --- a/fre/coveragerc +++ b/fre/coveragerc @@ -4,3 +4,12 @@ omit = fre/list/* fre/run/* fre/test/* + fre/tests/* + fre/analysis/tests/* + fre/app/generate_time_averages/tests/* + fre/app/regrid_xy/tests/* + fre/catalog/tests/* + fre/cmor/tests/* + fre/make/tests/* + fre/pp/tests/* + fre/yamltools/tests/* diff --git a/fre/pytest.ini b/fre/pytest.ini index 1842b6a8..80ef0c32 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -1,11 +1,11 @@ [pytest] testpaths = fre/tests + fre/analysis/tests + fre/app/generate_time_averages/tests + fre/app/regrid_xy/tests fre/catalog/tests fre/cmor/tests fre/make/tests fre/pp/tests fre/yamltools/tests - fre/analysis/tests - fre/app/generate_time_averages/tests - fre/app/regrid_xy/tests diff --git a/meta.yaml b/meta.yaml index 8ec674d0..54ac50fe 100644 --- a/meta.yaml +++ b/meta.yaml @@ -67,7 +67,7 @@ test: commands: - pylint --max-line-length 120 --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." # run pytest but ignore any tests that require compilation - - pytest --ignore=fre/make/tests/compilation --config-file=fre/pytest.ini --cov-report term-missing --cov-config=fre/coveragerc --cov=fre fre/ + - coverage run -m pytest --ignore=fre/make/tests/compilation --config-file=fre/pytest.ini --cov-report term-missing --cov-config=fre/coveragerc --cov=fre fre/ - fre --help - fre app --help - fre catalog --help From 41a99f2e98009a12bb855d035541151df8086c4c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 13 Jan 2025 17:00:49 -0500 Subject: [PATCH 45/45] whitespace newlines and fstrings thanks pylint you great friend --- environment.yml | 2 +- .../generate_time_averages/cdoTimeAverager.py | 2 +- .../frenctoolsTimeAverager.py | 2 +- fre/app/regrid_xy/tests/test_regrid_xy.py | 4 +- fre/cmor/cmor_finder.py | 15 +- fre/cmor/cmor_helpers.py | 11 +- fre/cmor/cmor_mixer.py | 138 +++++++++--------- fre/make/create_checkout_script.py | 2 +- fre/make/fremake.py | 8 +- .../compilation/test_run_fremake_builds.py | 5 +- fre/pp/checkout_script.py | 12 +- fre/pp/frepp.py | 4 +- fre/pp/install_script.py | 6 +- fre/pp/tests/test_configure_script_yaml.py | 5 +- fre/tests/test_fre_cli.py | 2 +- fre/tests/test_fre_cmor_cli.py | 2 +- fre/tests/test_fre_make_cli.py | 8 +- fre/tests/test_fre_pp_cli.py | 8 +- fre/yamltools/tests/test_combine_yamls.py | 2 +- 19 files changed, 118 insertions(+), 120 deletions(-) diff --git a/environment.yml b/environment.yml index 0a82e938..457379ba 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: fre_cli +name: fre-cli channels: - conda-forge - noaa-gfdl diff --git a/fre/app/generate_time_averages/cdoTimeAverager.py b/fre/app/generate_time_averages/cdoTimeAverager.py index d288bebb..9b15953a 100644 --- a/fre/app/generate_time_averages/cdoTimeAverager.py +++ b/fre/app/generate_time_averages/cdoTimeAverager.py @@ -20,7 +20,7 @@ def generate_timavg(self, infile=None, outfile=None): if self.var is not None: print(f'WARNING: variable specification (var={self.var})' + \ - f' not currently supported for cdo time averaging. ignoring!') + ' not currently supported for cdo time averaging. ignoring!') import cdo print(f'python-cdo version is {cdo.__version__}') diff --git a/fre/app/generate_time_averages/frenctoolsTimeAverager.py b/fre/app/generate_time_averages/frenctoolsTimeAverager.py index 32b9c2bb..8e6e943f 100644 --- a/fre/app/generate_time_averages/frenctoolsTimeAverager.py +++ b/fre/app/generate_time_averages/frenctoolsTimeAverager.py @@ -26,7 +26,7 @@ def generate_timavg(self, infile=None, outfile=None): if self.var is not None: print(f'WARNING: variable specification (var={self.var})' + \ - f' not currently supported for frenctols time averaging. ignoring!') + ' not currently supported for frenctols time averaging. ignoring!') if infile is None: print('ERROR: I need an input file, specify a value for the infile argument') diff --git a/fre/app/regrid_xy/tests/test_regrid_xy.py b/fre/app/regrid_xy/tests/test_regrid_xy.py index fa511b66..9df1871f 100644 --- a/fre/app/regrid_xy/tests/test_regrid_xy.py +++ b/fre/app/regrid_xy/tests/test_regrid_xy.py @@ -287,7 +287,7 @@ def test_success_tar_grid_spec_regrid_xy(capfd): rose_app_run_config.write( f'outputGridLat={NLAT}\n' ) rose_app_run_config.write( '\n' ) assert Path('./rose-app-run.conf').exists() - + rgxy_returncode = rgxy.regrid_xy( input_dir = WORK_YYYYMMDD_DIR, output_dir = TEST_OUT_DIR, @@ -431,7 +431,7 @@ def test_failure_wrong_datetime_regrid_xy(capfd): rose_app_run_config.write( f'outputGridLat={NLAT}\n' ) rose_app_run_config.write( '\n' ) assert Path('./rose-app-run.conf').exists() - + try: rgxy_returncode = rgxy.regrid_xy( input_dir = WORK_YYYYMMDD_DIR, diff --git a/fre/cmor/cmor_finder.py b/fre/cmor/cmor_finder.py index c0f3108b..c86c5635 100644 --- a/fre/cmor/cmor_finder.py +++ b/fre/cmor/cmor_finder.py @@ -23,7 +23,7 @@ def print_var_content( table_config_file = None, var_name = None): try: proj_table_vars=json.load(table_config_file) except Exception as exc: - raise Exception(f'problem getting proj_table_vars... WHY') + raise Exception('problem getting proj_table_vars... WHY') var_content = None try: @@ -44,7 +44,7 @@ def print_var_content( table_config_file = None, var_name = None): table_name = proj_table_vars["Header"].get('table_id').split(' ')[1] #print(f' table_name = {table_name}') except: - print(f'print_var_content) WARNING couldnt get header and table_name field') + print('print_var_content) WARNING couldnt get header and table_name field') pass if table_name is not None: @@ -74,7 +74,7 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v if json_table_configs is None: raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.') else: - print(f'(cmor_find_subtool) found content in json_table_config_dir')#: {json_table_configs}') + print('(cmor_find_subtool) found content in json_table_config_dir') var_list = None if json_var_list is not None: @@ -82,17 +82,17 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v var_list=json.load(var_list_file) if opt_var_name is None and var_list is None: - raise ValueError(f'(cmor_find_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') + raise ValueError('(cmor_find_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!') if opt_var_name is not None: - print(f'(cmor_find_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') + print('(cmor_find_subtool) opt_var_name is not None: looking for only ONE variables worth of info!') for json_table_config in json_table_configs: #print(f'(cmor_find_subtool) attempting to open {json_table_config}') with open( json_table_config, "r", encoding = "utf-8") as table_config_file: print_var_content(table_config_file, opt_var_name) elif var_list is not None: - print(f'(cmor_find_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') + print('(cmor_find_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!') for var in var_list: for json_table_config in json_table_configs: #print(f'(cmor_find_subtool) attempting to open {json_table_config}') @@ -100,6 +100,7 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v #print(f' var = {var}, var_list[{var}]={var_list[var]}') print_var_content(table_config_file, str(var_list[var])) else: - print(f'(FATAL) this line should be unreachable!!!') + print('(FATAL) this line should be unreachable!!!') + assert False return diff --git a/fre/cmor/cmor_helpers.py b/fre/cmor/cmor_helpers.py index bab9899e..de961728 100644 --- a/fre/cmor/cmor_helpers.py +++ b/fre/cmor/cmor_helpers.py @@ -20,7 +20,7 @@ def print_data_minmax(ds_variable = None, desc = None): pass print('----------------------------------------------------------------------------------------------------------') return - + def from_dis_gimme_dis(from_dis, gimme_dis): ''' @@ -56,9 +56,9 @@ def find_statics_file(bronx_file_path): def create_lev_bnds(bound_these = None, with_these = None): the_bnds = None assert len(with_these) == len(bound_these) + 1 - print(f'(create_lev_bnds) bound_these is... ') + print( '(create_lev_bnds) bound_these is... ') print(f' bound_these = \n{bound_these}') - print(f'(create_lev_bnds) with_these is... ') + print( '(create_lev_bnds) with_these is... ') print(f' with_these = \n{with_these}') @@ -66,7 +66,7 @@ def create_lev_bnds(bound_these = None, with_these = None): for i in range(0,len(bound_these)): the_bnds[i][0] = with_these[i] the_bnds[i][1] = with_these[i+1] - print(f'(create_lev_bnds) the_bnds is... ') + print( '(create_lev_bnds) the_bnds is... ') print(f' the_bnds = \n{the_bnds}') return the_bnds @@ -174,7 +174,7 @@ def create_tmp_dir(outdir, json_exp_config = None): try: outdir_from_exp_config = json.load(table_config_file)["outpath"] except: - print(f'(create_tmp_dir) WARNING could not read outdir from json_exp_config.' + print( '(create_tmp_dir) WARNING could not read outdir from json_exp_config.' ' the cmor module will throw a toothless warning' ) # assign an appropriate temporary working directory @@ -204,4 +204,3 @@ def create_tmp_dir(outdir, json_exp_config = None): raise OSError(f'(create_tmp_dir) problem creating tmp output directory {tmp_dir}. stop.') from exc return tmp_dir - diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py index 960e91aa..658ea58e 100755 --- a/fre/cmor/cmor_mixer.py +++ b/fre/cmor/cmor_mixer.py @@ -16,7 +16,7 @@ import cmor from .cmor_helpers import * -# ----- \start consts # TODO make this an input argument flag or smth. +# ----- \start consts # TODO make this an input argument flag or smth. DEBUG_MODE_RUN_ONE = True # ----- \end consts @@ -74,21 +74,21 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, ## figure out the coordinate/dimension names programmatically TODO # Attempt to read lat coordinates - print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lat') + print('(rewrite_netcdf_file_var) attempting to read coordinate, lat') lat = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lat") - print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lat_bnds') + print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lat_bnds') lat_bnds = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lat_bnds") - print(f'(rewrite_netcdf_file_var) attempting to read coordinate, lon') + print('(rewrite_netcdf_file_var) attempting to read coordinate, lon') lon = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lon") - print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds') + print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds') lon_bnds = from_dis_gimme_dis( from_dis = ds, gimme_dis = "lon_bnds") # read in time_coords + units - print(f'(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') + print('(rewrite_netcdf_file_var) attempting to read coordinate time, and units...') time_coords = from_dis_gimme_dis( from_dis = ds, gimme_dis = 'time' ) @@ -96,7 +96,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print(f" time_coord_units = {time_coord_units}") # read in time_bnds , if present - print(f'(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') + print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds') time_bnds = from_dis_gimme_dis( from_dis = ds, gimme_dis = 'time_bnds' ) @@ -116,7 +116,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # determine the vertical dimension by looping over netcdf variables vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}") - + # Check var_dim and vert_dim and assign lev if relevant. # error if vert_dim wrong given var_dim lev, lev_units = None, "1" #1 #"none" #None #"" @@ -130,7 +130,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lev_units = ds[vert_dim].units - + # the tripolar grid is designed to reduce distortions in ocean data brought on # by singularities (poles) being placed in oceans # the spherical lat/lons tend to already be computed in advance at GFDL, they're in "statics" @@ -150,28 +150,28 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, statics_file_path = find_statics_file(prev_path) print(f'(rewrite_netcdf_file_var) statics_file_path is {statics_file_path}') except Exception as exc: - print(f'(rewrite_netcdf_file_var) WARNING: pretty sure an ocean statics file is needed, but it could not be found.' - ' moving on and doing my best, but i am probably going to break' ) + print('(rewrite_netcdf_file_var) WARNING: an ocean statics file is needed, but it could not be found.\n' + ' moving on and doing my best, but i am probably going to break' ) raise Exception('(rewrite_netcdf_file_var) EXITING BC STATICS') from exc - print(f"(rewrite_netcdf_file_var) statics file found.") + print("(rewrite_netcdf_file_var) statics file found.") statics_file_name = Path(statics_file_path).name put_statics_file_here = str(Path(netcdf_file).parent) shutil.copy(statics_file_path, put_statics_file_here) del statics_file_path - + statics_file_path = put_statics_file_here + '/' + statics_file_name print(f'(rewrite_netcdf_file_var) statics file path is now: {statics_file_path}') # statics file read statics_ds = nc.Dataset(statics_file_path, 'r') - + # grab the lat/lon points, have shape (yh, xh) - print(f'(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell centers from statics file \n') + print('(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell centers from statics file \n') statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy() statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy() - + print('\n') print_data_minmax(statics_lat, "statics_lat") print_data_minmax(statics_lon, "statics_lon") @@ -179,12 +179,12 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # spherical lat and lon coords - print(f'(rewrite_netcdf_file_var) creating lat and lon variables in temp file \n') + print('(rewrite_netcdf_file_var) creating lat and lon variables in temp file \n') lat = ds.createVariable('lat', statics_lat.dtype, ('yh', 'xh') ) lon = ds.createVariable('lon', statics_lon.dtype, ('yh', 'xh') ) lat[:] = statics_lat[:] lon[:] = statics_lon[:] - + print('\n') print_data_minmax(lat[:], "lat") print_data_minmax(lon[:], "lon") @@ -192,24 +192,24 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # grab the corners of the cells, should have shape (yh+1, xh+1) - print(f'(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell corners from statics file \n') + print('(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell corners from statics file \n') lat_c = from_dis_gimme_dis(statics_ds,'geolat_c') lon_c = from_dis_gimme_dis(statics_ds,'geolon_c') - + print('\n') print_data_minmax(lat_c, "lat_c") print_data_minmax(lon_c, "lon_c") print('\n') - + # vertex - print(f'(rewrite_netcdf_file_var) creating vertex dimension\n') + print('(rewrite_netcdf_file_var) creating vertex dimension\n') vertex = 4 ds.createDimension('vertex', vertex) # lat and lon bnds - print(f'(rewrite_netcdf_file_var) creating lat and lon bnds from geolat and geolon of corners\n') + print('(rewrite_netcdf_file_var) creating lat and lon bnds from geolat and geolon of corners\n') lat_bnds = ds.createVariable('lat_bnds', lat_c.dtype, ('yh', 'xh', 'vertex') ) lat_bnds[:,:,0] = lat_c[1:,1:] # NE corner lat_bnds[:,:,1] = lat_c[1:,:-1] # NW corner @@ -221,7 +221,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, lon_bnds[:,:,1] = lon_c[1:,:-1] # NW corner lon_bnds[:,:,2] = lon_c[:-1,:-1] # SW corner lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner - + print('\n') print_data_minmax(lat_bnds[:], "lat_bnds") print_data_minmax(lon_bnds[:], "lon_bnds") @@ -229,7 +229,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # grab the h-point lat and lon - print(f'(rewrite_netcdf_file_var) reading yh, xh\n') + print('(rewrite_netcdf_file_var) reading yh, xh\n') yh = from_dis_gimme_dis(ds, 'yh') xh = from_dis_gimme_dis(ds, 'xh') @@ -242,8 +242,8 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, xh_dim = len(xh) # read the q-point native-grid lat lon points - print(f'(rewrite_netcdf_file_var) reading yq, xq from statics file \n') - yq = from_dis_gimme_dis(statics_ds, 'yq') + print('(rewrite_netcdf_file_var) reading yq, xq from statics file \n') + yq = from_dis_gimme_dis(statics_ds, 'yq') xq = from_dis_gimme_dis(statics_ds, 'xq') print('\n') @@ -253,9 +253,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, assert yh_dim == (len(yq)-1) assert xh_dim == (len(xq)-1) - + # create h-point bounds from the q-point lat lons - print(f'(rewrite_netcdf_file_var) creating yh_bnds, xh_bnds from yq, xq\n') + print('(rewrite_netcdf_file_var) creating yh_bnds, xh_bnds from yq, xq\n') yh_bnds = ds.createVariable('yh_bnds', yq.dtype, ( 'yh', 'nv' ) ) for i in range(0,yh_dim): @@ -277,7 +277,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, print('\n') - # now we set up the cmor module object + # now we set up the cmor module object # initialize CMOR cmor.setup( netcdf_file_action = cmor.CMOR_APPEND,#.CMOR_PRESERVE, # @@ -287,17 +287,17 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, create_subdirectories = 1 ) - + # read experiment configuration file print(f"(rewrite_netcdf_file_var) cmor is opening: json_exp_config = {json_exp_config}") cmor.dataset_json(json_exp_config) - + # load CMOR table print(f"(rewrite_netcdf_file_var) cmor is loading+setting json_table_config = {json_table_config}") loaded_cmor_table_cfg = cmor.load_table(json_table_config) cmor.set_table(loaded_cmor_table_cfg) - - + + # if ocean tripolar grid, we need the CMIP grids configuration file. load it but don't set the table yet. json_grids_config, loaded_cmor_grids_cfg = None, None if process_tripolar_data: @@ -308,37 +308,37 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, - + # setup cmor latitude axis if relevant cmor_y = None if process_tripolar_data: - print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!') + print('(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!') cmor_y = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees") elif lat is None : - print(f'(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_y') + print('(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_y') else: - print(f'(rewrite_netcdf_file_var) assigning cmor_y') + print('(rewrite_netcdf_file_var) assigning cmor_y') if lat_bnds is None: cmor_y = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N") else: cmor_y = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N") - print(f' DONE assigning cmor_y') + print(' DONE assigning cmor_y') # setup cmor longitude axis if relevant cmor_x = None if process_tripolar_data: - print(f'(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!') + print('(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!') cmor_x = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees") elif lon is None : - print(f'(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_x') + print('(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_x') else: - print(f'(rewrite_netcdf_file_var) assigning cmor_x') + print('(rewrite_netcdf_file_var) assigning cmor_x') cmor_x = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E") if lon_bnds is None: cmor_x = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E") else: cmor_x = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E") - print(f' DONE assigning cmor_x') + print(' DONE assigning cmor_x') cmor_grid = None if process_tripolar_data: @@ -347,26 +347,26 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, latitude = lat[:], longitude = lon[:], latitude_vertices = lat_bnds[:], longitude_vertices = lon_bnds[:] ) - + # now that we are done with setting the grid, we can go back to the usual approach cmor.set_table(loaded_cmor_table_cfg) # setup cmor time axis if relevant cmor_time = None - print(f'(rewrite_netcdf_file_var) assigning cmor_time') + print('(rewrite_netcdf_file_var) assigning cmor_time') try: #if vert_dim != 'landuse': - print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" + print( "(rewrite_netcdf_file_var) Executing cmor.axis('time', \n" f" coord_vals = \n{time_coords}, \n" f" cell_bounds = time_bnds, units = {time_coord_units}) ") - print(f'(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') + print('(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, cell_bounds = time_bnds, units = time_coord_units) except ValueError as exc: #else: - print(f"(rewrite_netcdf_file_var) cmor_time = cmor.axis('time', \n" + print("(rewrite_netcdf_file_var) cmor_time = cmor.axis('time', \n" " coord_vals = time_coords, units = time_coord_units)") - print(f'(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') + print('(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...') cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units) - print(f' DONE assigning cmor_time') + print(' DONE assigning cmor_time') # other vertical-axis-relevant initializations @@ -378,10 +378,10 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, # set cmor vertical axis if relevant cmor_z = None if lev is not None: - print(f'(rewrite_netcdf_file_var) assigning cmor_z') + print('(rewrite_netcdf_file_var) assigning cmor_z') if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]: - print(f'(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') + print('(rewrite_netcdf_file_var) non-hybrid sigma coordinate case') if vert_dim.lower() != "landuse": cmor_vert_dim_name = vert_dim cmor_z = cmor.axis( cmor_vert_dim_name, @@ -450,16 +450,16 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, f'(rewrite_netcdf_file_var) ierr_b after calling cmor_zfactor: {ierr_b}' ) axis_ids = [] if cmor_time is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') + print('(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...') axis_ids.append(cmor_time) print(f' axis_ids now = {axis_ids}') # might there need to be a conditional check for tripolar ocean data here as well? TODO if cmor_y is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_y to axis_ids list...') + print('(rewrite_netcdf_file_var) appending cmor_y to axis_ids list...') axis_ids.append(cmor_y) print(f' axis_ids now = {axis_ids}') if cmor_x is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_x to axis_ids list...') + print('(rewrite_netcdf_file_var) appending cmor_x to axis_ids list...') axis_ids.append(cmor_x) print(f' axis_ids now = {axis_ids}') @@ -468,29 +468,29 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, axis_ids = axis_ids, #[cmor_time, cmor_y, cmor_x], units = "Pa" ) save_ps = True - print(f' DONE assigning cmor_z') + print(' DONE assigning cmor_z') axes = [] if cmor_time is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_time to axes list...') + print('(rewrite_netcdf_file_var) appending cmor_time to axes list...') axes.append(cmor_time) print(f' axes now = {axes}') if cmor_z is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_z to axes list...') + print('(rewrite_netcdf_file_var) appending cmor_z to axes list...') axes.append(cmor_z) print(f' axes now = {axes}') - - if process_tripolar_data: + + if process_tripolar_data: axes.append(cmor_grid) else: if cmor_y is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_y to axes list...') + print('(rewrite_netcdf_file_var) appending cmor_y to axes list...') axes.append(cmor_y) print(f' axes now = {axes}') if cmor_x is not None: - print(f'(rewrite_netcdf_file_var) appending cmor_x to axes list...') + print('(rewrite_netcdf_file_var) appending cmor_x to axes list...') axes.append(cmor_x) print(f' axes now = {axes}') @@ -501,7 +501,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None, positive = proj_table_vars["variable_entry"] [target_var] ["positive"] print(f"(rewrite_netcdf_file_var) positive = {positive}") - + cmor_var = cmor.variable(target_var, units, axes, positive = positive) # Write the output to disk @@ -659,8 +659,8 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None, Path(nc_ps_file_work).unlink() if DEBUG_MODE_RUN_ONE: - print(f'WARNING: DEBUG_MODE_RUN_ONE is True!!!!') - print(f'WARNING: done processing one file!!!') + print('WARNING: DEBUG_MODE_RUN_ONE is True!!!!') + print('WARNING: done processing one file!!!') break @@ -693,10 +693,10 @@ def cmor_run_subtool( indir = None, ''' # check req'd inputs if None in [indir, json_var_list, json_table_config, json_exp_config, outdir]: - raise ValueError(f'(cmor_run_subtool) all input arguments except opt_var_name are required!\n' + raise ValueError( '(cmor_run_subtool) all input arguments except opt_var_name are required!\n' ' [indir, json_var_list, json_table_config, json_exp_config, outdir] = \n' f' [{indir}, {json_var_list}, {json_table_config}, ' - ' {json_exp_config}, {outdir}]' ) + f' {json_exp_config}, {outdir}]' ) # open CMOR table config file print( '(cmor_run_subtool) loading json_table_config = \n' @@ -749,7 +749,7 @@ def cmor_run_subtool( indir = None, ' to the opt_var_name argument.') continue print('\n') - + # it is in there, get the name of the data inside the netcdf file. target_var = var_list[local_var] # often equiv to local_var but not necessarily. if local_var != target_var: @@ -786,6 +786,6 @@ def cmor_run_subtool( indir = None, ) if DEBUG_MODE_RUN_ONE: - print(f'WARNING: DEBUG_MODE_RUN_ONE is True. breaking var_list loop') + print('WARNING: DEBUG_MODE_RUN_ONE is True. breaking var_list loop') break return 0 diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 8f5ef379..457c2e73 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -86,7 +86,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec else: return else: - print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") + print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") if run: try: subprocess.run(args=[src_dir+"/checkout.sh"], check=True) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 0593445c..a447cc46 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -46,7 +46,7 @@ def make_cli(): type = str, help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", - multiple = True, + multiple = True, type = str, help = TARGET_OPT_HELP, required = True) @@ -121,7 +121,7 @@ def create_checkout(yamlfile, platform, target, no_parallel_checkout, jobs, exec yamlfile, platform, target, no_parallel_checkout, jobs, execute, verbose) @make_cli.command -@click.option("-y", +@click.option("-y", "--yamlfile", type = str, help = YAMLFILE_OPT_HELP, @@ -185,14 +185,14 @@ def create_compile(yamlfile, platform, target, jobs, parallel, execute, verbose) "--yamlfile", type = str, help = YAMLFILE_OPT_HELP, - required = True) + required = True) @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", - multiple = True, + multiple = True, type = str, help = TARGET_OPT_HELP, required = True) diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 3967323a..5b2eba86 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -31,7 +31,7 @@ # test building the null model using gnu compilers -@pytest.mark.skip(reason="EXPECTED TO FAIL ON WORKSTATION REMOVE THIS LINE PREMERGE") +@pytest.mark.skip(reason="fails on workstation") def test_run_fremake_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH @@ -39,7 +39,7 @@ def test_run_fremake_serial_compile(): assert Path(f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() # same test with a parallel build -@pytest.mark.skip(reason="EXPECTED TO FAIL ON WORKSTATION REMOVE THIS LINE PREMERGE") +@pytest.mark.skip(reason="fails on workstation") def test_run_fremake_multijob_compile(): ''' test run-fremake parallel compile with gnu''' os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH @@ -52,4 +52,3 @@ def test_run_fremake_container_build(): ''' checks image creation for the container build''' run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, True, VERBOSE) assert Path("null_model_full-debug.sif").exists() - diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index c374b6c2..e2f64d6a 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -1,8 +1,8 @@ ''' -Description: Checkout script which accounts for 4 different scenarios: +Description: Checkout script which accounts for 4 different scenarios: 1. branch not given, folder does not exist, -2. branch given, folder does not exist, -3. branch not given, folder exists, +2. branch given, folder does not exist, +3. branch not given, folder exists, 4. branch given and folder exists ''' import os @@ -48,7 +48,7 @@ def checkout_template(experiment = None, platform = None, target = None, branch checkout_exists = os.path.isdir(f'{directory}/{name}') if not checkout_exists: # scenarios 1+2, checkout doesn't exist, branch specified (or not) - print(f'(checkout_script) checkout does not yet exist; will create now') + print('(checkout_script) checkout does not yet exist; will create now') clone_output = subprocess.run( ['git', 'clone','--recursive', f'--branch={git_clone_branch_arg}', FRE_WORKFLOWS_URL, f'{directory}/{name}'], @@ -73,14 +73,14 @@ def checkout_template(experiment = None, platform = None, target = None, branch else: print(f"(checkout_script) ERROR: checkout exists ('{directory}/{name}') and does not match '{git_clone_branch_arg}'") print(f"(checkout_script) ERROR: current branch is '{current_branch}', current tag-describe is '{current_tag}'") - os.chdir(go_back_here) + os.chdir(go_back_here) raise ValueError('(checkout_script) neither tag nor branch matches the git clone branch arg') #exit(1) # make sure we are back where we should be if os.getcwd() != go_back_here: os.chdir(go_back_here) - return + return ############################################# diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index ef9e6ff3..910a09be 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -192,9 +192,9 @@ def configure_xml(xml, platform, target, experiment, do_analysis, historydir, re help="Time whose history files are ready") def wrapper(experiment, platform, target, config_file, branch, time): """ - Execute fre pp steps in order """ - print(f'(frepp.wrapper) about to foward context to wrapper.run_all_fre_pp_steps via click...') + print('(frepp.wrapper) about to foward context to wrapper.run_all_fre_pp_steps via click...') wrapper_script.run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time) - print(f'(frepp.wrapper) done fowarding context to wrapper.run_all_fre_pp_steps via click.') + print('(frepp.wrapper) done fowarding context to wrapper.run_all_fre_pp_steps via click.') @pp_cli.command() @click.option("-e", "--experiment", type=str, diff --git a/fre/pp/install_script.py b/fre/pp/install_script.py index ea7f2add..05558fa5 100644 --- a/fre/pp/install_script.py +++ b/fre/pp/install_script.py @@ -27,10 +27,10 @@ def install_subtool(experiment, platform, target): if installed_def == source_def: print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") else: - print(f"ERROR: Please remove installed workflow with 'cylc clean {name}' or move the workflow run directory '{install_dir}'") - raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") #exit(1) + print(f"ERROR: Please remove installed workflow with 'cylc clean {name}'" + " or move the workflow run directory '{install_dir}'") + raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") else: print(f"NOTE: About to install workflow into ~/cylc-run/{name}") cmd = f"cylc install --no-run-name {name}" subprocess.run(cmd, shell=True, check=True) - diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index a825275b..24232710 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -22,7 +22,7 @@ def test_combinedyaml_exists(): assert Path(f"{TEST_DIR}/{TEST_YAML}").exists() def test_configure_script(): - """ + """ Tests success of confgure yaml script Creates rose-suite, regrid rose-app, remap rose-app TO-DO: will break this up for better tests @@ -48,8 +48,7 @@ def test_configure_script(): Path(f"{OUT_DIR}/rose-suite.conf").exists(), Path(f"{OUT_DIR}/app/regrid-xy/rose-app.conf").exists(), Path(f"{OUT_DIR}/app/remap-pp-components/rose-app.conf").exists() ]) - + def test_cleanup(): shutil.rmtree(f"{TEST_DIR}/configure_yaml_out") assert not Path(f"{TEST_DIR}/configure_yaml_out").exists() - diff --git a/fre/tests/test_fre_cli.py b/fre/tests/test_fre_cli.py index d4d758ce..6cdb71f2 100644 --- a/fre/tests/test_fre_cli.py +++ b/fre/tests/test_fre_cli.py @@ -25,7 +25,7 @@ def test_cli_fre_option_dne(): def test_fre_version(): ''' module import flavor of below cli test ''' assert '2025.01' == fre.version - + def test_cli_fre_version(): ''' fre --version ''' result = runner.invoke(fre.fre, args='--version') diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index 195965f4..30a578e3 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -60,7 +60,7 @@ def test_setup_test_files(capfd): if Path(full_copied_nc_filepath).exists(): Path(full_copied_nc_filepath).unlink() assert not Path(full_copied_nc_filepath).exists() - + shutil.copy(Path(original_nc_file), Path(full_copied_nc_filepath)) assert (Path(full_copied_nc_filepath).exists()) diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py index a681964e..4d0bde8e 100644 --- a/fre/tests/test_fre_make_cli.py +++ b/fre/tests/test_fre_make_cli.py @@ -30,7 +30,7 @@ def test_cli_fre_make_create_checkout_baremetal(): # remove the created script to re-create it, if it exists #if Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").exists(): # Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").unlink() - + # Set paths and click options yamlfile = Path("fre/make/tests/AM5_example/") platform = "ncrc5.intel23" @@ -42,13 +42,13 @@ def test_cli_fre_make_create_checkout_baremetal(): # Set HOME for modelRoot location (output location) in fre make old_home = os.environ["HOME"] os.environ["HOME"]=str(Path(OUT_PATH)) - + # run create-checkout result = runner.invoke(fre.fre, args=["make", "create-checkout", "-y", f"{yamlfile}/am5.yaml", "-p", platform, "-t", target]) os.environ["HOME"] = old_home - # Check for successful command, creation of checkout script, and that script is executable + # Check for successful command, creation of checkout script, and that script is executable # os.access - checks is file has specific access mode, os.X_OK - checks executable permission assert all ([result.exit_code == 0, Path(f"{OUT_PATH}/fremake_canopy/test/am5/src/checkout.sh").exists(), @@ -75,7 +75,7 @@ def test_cli_fre_make_create_checkout_container(): assert all ([result.exit_code == 0, Path(f"tmp/{platform}/checkout.sh").exists(), os.access(Path(f"tmp/{platform}/checkout.sh"), os.X_OK) == False ]) - + def test_cli_fre_make_create_checkout_cleanup(): ''' make sure the checked out code doesnt stick around to mess up another pytest call ''' assert Path(OUT_PATH).exists() diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index e60d012d..6249ae5d 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -48,12 +48,12 @@ def test_cli_fre_pp_checkout_case(): directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' if Path(directory).exists(): shutil.rmtree(directory) - result = runner.invoke(fre.fre, args=["pp", "checkout", - "-e", "FOO", - "-p", "BAR", + result = runner.invoke(fre.fre, args=["pp", "checkout", + "-e", "FOO", + "-p", "BAR", "-t", "BAZ"] ) assert all( [ result.exit_code == 0, - Path(directory).exists()] ) + Path(directory).exists()] ) #-- fre pp configure-xml def test_cli_fre_pp_configure_xml(): diff --git a/fre/yamltools/tests/test_combine_yamls.py b/fre/yamltools/tests/test_combine_yamls.py index 7df6eb36..ae0cc3f2 100644 --- a/fre/yamltools/tests/test_combine_yamls.py +++ b/fre/yamltools/tests/test_combine_yamls.py @@ -83,7 +83,7 @@ def test_combined_compileyaml_validation(): """ combined_yamlfile =f"{COMP_OUT_DIR}/combined-{COMP_EXPERIMENT}.yaml" schema_file = os.path.join(SCHEMA_DIR, "fre_make.json") - + with open(combined_yamlfile,'r') as cf: yml = yaml.safe_load(cf)