From 9ecc7079b5f4ff9a68bd1fac2061fa21696f608e Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 11:33:42 -0500
Subject: [PATCH 01/23] add --no-wait and --pause options to fre pp run,
 dinking around with logger and where to first make one. fre.py?

---
 fre/fre.py           |  2 ++
 fre/pp/frepp.py      | 10 ++++++++--
 fre/pp/run_script.py | 22 ++++++++++++++++++----
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/fre/fre.py b/fre/fre.py
index 249a06c7..d09715a3 100644
--- a/fre/fre.py
+++ b/fre/fre.py
@@ -50,6 +50,8 @@
 
 def fre():
     ''' entry point function to subgroup functions '''
+    print('hello from fre.py fre()')
 
 if __name__ == '__main__':
+    print('hello from fre.py __main__')
     fre()
diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py
index 910a09be..4d32c63e 100644
--- a/fre/pp/frepp.py
+++ b/fre/pp/frepp.py
@@ -45,9 +45,15 @@ def status(experiment, platform, target):
 @click.option("-t", "--target", type=str,
               help="Target name",
               required=True)
-def run(experiment, platform, target):
+@click.option("--pause", is_flag=True, default=False,
+              help="Pause the workflow immediately on start up",
+              required=False)
+@click.option("--no_wait", is_flag=True, default=False,
+              help="after submission, do not wait to ping the scheduler and confirm success",
+              required=False)
+def run(experiment, platform, target, pause, no_wait):
     """ - Run PP configuration"""
-    run_script.pp_run_subtool(experiment, platform, target)
+    run_script.pp_run_subtool(experiment, platform, target, pause, no_wait)
 
 # fre pp validate
 @pp_cli.command()
diff --git a/fre/pp/run_script.py b/fre/pp/run_script.py
index 982b7691..72fc65f6 100644
--- a/fre/pp/run_script.py
+++ b/fre/pp/run_script.py
@@ -3,7 +3,8 @@
 import subprocess
 import time
 
-def pp_run_subtool(experiment = None, platform = None, target = None):
+def pp_run_subtool(experiment = None, platform = None, target = None,
+                   pause = False, no_wait = False):
     """
     Start or restart the Cylc workflow identified by:
     <experiment>__<platform>__<target>
@@ -21,16 +22,29 @@ def pp_run_subtool(experiment = None, platform = None, target = None):
         return
 
     # If not running, start it
-    cmd = f"cylc play {name}"
+    cmd  = "cylc play"
+    if pause:
+        cmd+=" --pause"
+    cmd +=f" {name}"
     subprocess.run(cmd, shell=True, check=True)
 
-    # wait 30 seconds for the scheduler to come up; then confirm
+    # not interested in the confirmation? gb2work now
+    if no_wait:
+        return
+    
+    # give the scheduler 30 seconds of peace before we hound it
     print("Workflow started; waiting 30 seconds to confirm")
     time.sleep(30)
-    result = subprocess.run(['cylc', 'scan', '--name', f"^{name}$"], capture_output=True).stdout.decode('utf-8')
 
+    # confirm the scheduler came up. note the regex surrounding {name} for start/end of a string to avoid glob matches
+    result = subprocess.run(
+        ['cylc', 'scan', '--name', f"^{name}$"],
+        capture_output = True ).stdout.decode('utf-8')
+    
     if not len(result):
         raise Exception('Cylc scheduler was started without error but is not running after 30 seconds')
 
+    print(result)
+    
 if __name__ == "__main__":
     pp_run_subtool()

From 5197ba95040a65c2854f8fd0329c4ab6152476b2 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 12:39:39 -0500
Subject: [PATCH 02/23] add verbose flag to base fre command and define
 fre_logger in fre/fre.py. add single logger.info line inlieu of print
 statement for commit + push to github for init CI/CD attempt

---
 fre/cmor/cmor_mixer.py | 13 ++++++++-----
 fre/fre.py             | 28 ++++++++++++++++------------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py
index 658ea58e..72d37388 100755
--- a/fre/cmor/cmor_mixer.py
+++ b/fre/cmor/cmor_mixer.py
@@ -4,11 +4,14 @@
 see README.md for additional information on `fre cmor run` (cmor_mixer.py) usage
 '''
 
-import os
 import json
+import logging
+fre_logger = logging.getLogger(__name__)
+
+import os
+from pathlib import Path
 import shutil
 import subprocess
-from pathlib import Path
 
 import numpy as np
 
@@ -698,9 +701,9 @@ def cmor_run_subtool( indir = None,
                          f'                   [{indir}, {json_var_list}, {json_table_config}, '
                          f'                   {json_exp_config}, {outdir}]' )
 
-    # open CMOR table config file
-    print( '(cmor_run_subtool) loading json_table_config = \n'
-           f'                      {json_table_config}\n'                     )
+    ## open CMOR table config file
+    fre_logger.info( 'loading json_table_config = \n'
+           f'    {json_table_config}\n'                     )
     try:
         with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
             proj_table_vars = json.load(table_config_file)
diff --git a/fre/fre.py b/fre/fre.py
index d09715a3..943557a2 100644
--- a/fre/fre.py
+++ b/fre/fre.py
@@ -12,7 +12,8 @@
 import click
 from .lazy_group import LazyGroup
 
-
+import logging
+fre_logger = logging.getLogger(__name__)
 
 # versioning, turn xxxx.y into xxxx.0y
 version_unexpanded = importlib.metadata.version('fre-cli')
@@ -23,6 +24,13 @@
     version_minor = version_unexpanded_split[1]
 version = version_unexpanded_split[0] + '.' + version_minor
 
+
+@click.version_option(
+    package_name = "fre-cli",
+    version=version
+)
+
+
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
@@ -38,20 +46,16 @@
                        "cmor": ".cmor.frecmor.cmor_cli",
                        "analysis": ".analysis.freanalysis.analysis_cli"},
     help = click.style(
-        "'fre' is the main CLI click group that houses the other tool groups as lazy subcommands.",
+        "'fre' is the main CLI click group. It houses the other tool groups as lazy subcommands.",
         fg='cyan')
 )
-
-
-@click.version_option(
-    package_name = "fre-cli",
-    version=version
-)
-
-def fre():
+@click.option('-v', '--verbose', is_flag=True,
+              default=False, help="set logging verbosity higher",
+              required=False)
+def fre(verbose):
     ''' entry point function to subgroup functions '''
-    print('hello from fre.py fre()')
+    if verbose:
+        logging.basicConfig(level=logging.INFO)
 
 if __name__ == '__main__':
-    print('hello from fre.py __main__')
     fre()

From 3416118d369b0643cfed4d8e5a2d885bcc8eb07d Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 13:17:18 -0500
Subject: [PATCH 03/23] print --> logger.info or logger.warn

---
 fre/cmor/cmor_mixer.py    | 295 +++++++++++++++++++-------------------
 fre/fre.py                |   2 +
 fre/pp/checkout_script.py |   2 +-
 3 files changed, 150 insertions(+), 149 deletions(-)

diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py
index 72d37388..f0cc301c 100755
--- a/fre/cmor/cmor_mixer.py
+++ b/fre/cmor/cmor_mixer.py
@@ -45,72 +45,72 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         json_table_config: string, representing path to json configuration file holding variable names for a given table.
                            proj_table_vars is read from this file, but both are passed anyways.
     '''
-    print('\n\n-------------------------- START rewrite_netcdf_file_var call -----')
-    print( "(rewrite_netcdf_file_var) input data: " )
-    print(f"                              local_var   =  {local_var}" )
-    print(f"                              target_var = {target_var}")
+    fre_logger.info('\n\n-------------------------- START rewrite_netcdf_file_var call -----')
+    fre_logger.info( "input data: " )
+    fre_logger.info(f"    local_var   =  {local_var}" )
+    fre_logger.info(f"    target_var = {target_var}")
 
 
     # open the input file
-    print(f"(rewrite_netcdf_file_var) opening {netcdf_file}" )
+    fre_logger.info(f" opening {netcdf_file}" )
     ds = nc.Dataset(netcdf_file,'r+')#'a')
 
 
     # ocean grids are not implemented yet.
-    print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition')
+    fre_logger.info( ' checking input netcdf file for oceangrid condition')
     uses_ocean_grid = check_dataset_for_ocean_grid(ds)
     if uses_ocean_grid:
-        print('(rewrite_netcdf_file_var) OH BOY you have a file on the native tripolar grid...\n'
-              '                          ... this is gonna be fun!' )
+        fre_logger.info(' OH BOY you have a file on the native tripolar grid...\n'
+              '... this is gonna be fun!' )
 
     # try to read what coordinate(s) we're going to be expecting for the variable
     expected_mip_coord_dims = None
     try:
         expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"]
-        print( '(rewrite_netcdf_file_var) i am hoping to find data for the following coordinate dimensions:\n'
-              f'                          expected_mip_coord_dims = {expected_mip_coord_dims}\n' )
+        fre_logger.info( ' i am hoping to find data for the following coordinate dimensions:\n'
+              f'    expected_mip_coord_dims = {expected_mip_coord_dims}\n' )
     except Exception as exc:
-        print(f'(rewrite_netcdf_file_var) WARNING could not get expected coordinate dimensions for {target_var}. '
-               '                          in proj_table_vars file {json_table_config}. \n exc = {exc}')
+        fre_logger.warning(f' could not get expected coordinate dimensions for {target_var}. '
+               '    in proj_table_vars file {json_table_config}. \n exc = {exc}')
 
 
     ## figure out the coordinate/dimension names programmatically TODO
 
     # Attempt to read lat coordinates
-    print('(rewrite_netcdf_file_var) attempting to read coordinate, lat')
+    fre_logger.info(' attempting to read coordinate, lat')
     lat = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lat")
-    print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lat_bnds')
+    fre_logger.info(' attempting to read coordinate BNDS, lat_bnds')
     lat_bnds = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lat_bnds")
-    print('(rewrite_netcdf_file_var) attempting to read coordinate, lon')
+    fre_logger.info(' attempting to read coordinate, lon')
     lon = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lon")
-    print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, lon_bnds')
+    fre_logger.info(' attempting to read coordinate BNDS, lon_bnds')
     lon_bnds = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lon_bnds")
 
     # read in time_coords + units
-    print('(rewrite_netcdf_file_var) attempting to read coordinate time, and units...')
+    fre_logger.info(' attempting to read coordinate time, and units...')
     time_coords = from_dis_gimme_dis( from_dis = ds,
                                       gimme_dis = 'time' )
 
     time_coord_units = ds["time"].units
-    print(f"                          time_coord_units = {time_coord_units}")
+    fre_logger.info(f"                          time_coord_units = {time_coord_units}")
 
     # read in time_bnds , if present
-    print('(rewrite_netcdf_file_var) attempting to read coordinate BNDS, time_bnds')
+    fre_logger.info(' attempting to read coordinate BNDS, time_bnds')
     time_bnds = from_dis_gimme_dis( from_dis = ds,
                                     gimme_dis = 'time_bnds' )
 
     # read the input variable data, i believe
-    print(f'(rewrite_netcdf_file_var) attempting to read variable data, {target_var}')
+    fre_logger.info(f' attempting to read variable data, {target_var}')
     var = from_dis_gimme_dis( from_dis = ds,
                               gimme_dis = target_var )
 
     # grab var_dim
     var_dim = len(var.shape)
-    print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}")
+    fre_logger.info(f" var_dim = {var_dim}, local_var = {local_var}")
 
     ## Check var_dim
     #if var_dim not in [3, 4]:
@@ -118,7 +118,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
     # determine the vertical dimension by looping over netcdf variables
     vert_dim = get_vertical_dimension(ds, target_var) # returns int( 0 ) if not present
-    print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}")
+    fre_logger.info(f" Vertical dimension of {target_var}: {vert_dim}")
 
     # Check var_dim and vert_dim and assign lev if relevant.
     # error if vert_dim wrong given var_dim
@@ -149,70 +149,70 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
         # resolve location of statics file required for this processing.
         try:
-            print(f'(rewrite_netcdf_file_var) netcdf_file is {netcdf_file}')
+            fre_logger.info(f' netcdf_file is {netcdf_file}')
             statics_file_path = find_statics_file(prev_path)
-            print(f'(rewrite_netcdf_file_var) statics_file_path is {statics_file_path}')
+            fre_logger.info(f' statics_file_path is {statics_file_path}')
         except Exception as exc:
-            print('(rewrite_netcdf_file_var) WARNING: an ocean statics file is needed, but it could not be found.\n'
-                  '                                   moving on and doing my best, but i am probably going to break' )
-            raise Exception('(rewrite_netcdf_file_var) EXITING BC STATICS') from exc
+            fre_logger.warning(' an ocean statics file is needed, but it could not be found.\n'
+                  '   moving on and doing my best, but i am probably going to break' )
+            raise Exception(' EXITING BC STATICS') from exc
 
-        print("(rewrite_netcdf_file_var) statics file found.")
+        fre_logger.info(" statics file found.")
         statics_file_name = Path(statics_file_path).name
         put_statics_file_here = str(Path(netcdf_file).parent)
         shutil.copy(statics_file_path, put_statics_file_here)
         del statics_file_path
 
         statics_file_path = put_statics_file_here + '/' + statics_file_name
-        print(f'(rewrite_netcdf_file_var) statics file path is now: {statics_file_path}')
+        fre_logger.info(f' statics file path is now: {statics_file_path}')
 
         # statics file read
         statics_ds = nc.Dataset(statics_file_path, 'r')
 
 
         # grab the lat/lon points, have shape (yh, xh)
-        print('(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell centers from statics file \n')
+        fre_logger.info(' reading geolat and geolon coordinates of cell centers from statics file \n')
         statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy()
         statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy()
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(statics_lat, "statics_lat")
         print_data_minmax(statics_lon, "statics_lon")
-        print('\n')
+        fre_logger.info('\n')
 
 
         # spherical lat and lon coords
-        print('(rewrite_netcdf_file_var) creating lat and lon variables in temp file \n')
+        fre_logger.info(' creating lat and lon variables in temp file \n')
         lat = ds.createVariable('lat', statics_lat.dtype, ('yh', 'xh') )
         lon = ds.createVariable('lon', statics_lon.dtype, ('yh', 'xh') )
         lat[:] = statics_lat[:]
         lon[:] = statics_lon[:]
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(lat[:], "lat")
         print_data_minmax(lon[:], "lon")
-        print('\n')
+        fre_logger.info('\n')
 
 
         # grab the corners of the cells, should have shape (yh+1, xh+1)
-        print('(rewrite_netcdf_file_var) reading geolat and geolon coordinates of cell corners from statics file \n')
+        fre_logger.info(' reading geolat and geolon coordinates of cell corners from statics file \n')
         lat_c = from_dis_gimme_dis(statics_ds,'geolat_c')
         lon_c = from_dis_gimme_dis(statics_ds,'geolon_c')
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(lat_c, "lat_c")
         print_data_minmax(lon_c, "lon_c")
-        print('\n')
+        fre_logger.info('\n')
 
 
         # vertex
-        print('(rewrite_netcdf_file_var) creating vertex dimension\n')
+        fre_logger.info(' creating vertex dimension\n')
         vertex = 4
         ds.createDimension('vertex', vertex)
 
 
         # lat and lon bnds
-        print('(rewrite_netcdf_file_var) creating lat and lon bnds from geolat and geolon of corners\n')
+        fre_logger.info(' creating lat and lon bnds from geolat and geolon of corners\n')
         lat_bnds = ds.createVariable('lat_bnds', lat_c.dtype, ('yh', 'xh', 'vertex') )
         lat_bnds[:,:,0] = lat_c[1:,1:] # NE corner
         lat_bnds[:,:,1] = lat_c[1:,:-1] # NW corner
@@ -225,40 +225,40 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         lon_bnds[:,:,2] = lon_c[:-1,:-1] # SW corner
         lon_bnds[:,:,3] = lon_c[:-1,1:] # SE corner
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(lat_bnds[:], "lat_bnds")
         print_data_minmax(lon_bnds[:], "lon_bnds")
-        print('\n')
+        fre_logger.info('\n')
 
 
         # grab the h-point lat and lon
-        print('(rewrite_netcdf_file_var) reading yh, xh\n')
+        fre_logger.info(' reading yh, xh\n')
         yh = from_dis_gimme_dis(ds, 'yh')
         xh = from_dis_gimme_dis(ds, 'xh')
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(yh[:], "yh")
         print_data_minmax(xh[:], "xh")
-        print('\n')
+        fre_logger.info('\n')
 
         yh_dim = len(yh)
         xh_dim = len(xh)
 
         # read the q-point native-grid lat lon points
-        print('(rewrite_netcdf_file_var) reading yq, xq from statics file \n')
+        fre_logger.info(' reading yq, xq from statics file \n')
         yq = from_dis_gimme_dis(statics_ds, 'yq')
         xq = from_dis_gimme_dis(statics_ds, 'xq')
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(yq, "yq")
         print_data_minmax(xq, "xq")
-        print('\n')
+        fre_logger.info('\n')
 
         assert yh_dim == (len(yq)-1)
         assert xh_dim == (len(xq)-1)
 
         # create h-point bounds from the q-point lat lons
-        print('(rewrite_netcdf_file_var) creating yh_bnds, xh_bnds from yq, xq\n')
+        fre_logger.info(' creating yh_bnds, xh_bnds from yq, xq\n')
 
         yh_bnds = ds.createVariable('yh_bnds', yq.dtype, ( 'yh', 'nv' ) )
         for i in range(0,yh_dim):
@@ -270,14 +270,14 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
             xh_bnds[i,0] = xq[i]
             xh_bnds[i,1] = xq[i+1]
             if i%200 == 0:
-                print(f' AFTER assignment: xh_bnds[{i}][0] = {xh_bnds[i][0]}')
-                print(f' AFTER assignment: xh_bnds[{i}][1] = {xh_bnds[i][1]}')
-                print(f'             type(xh_bnds[{i}][1]) = {type(xh_bnds[i][1])}')
+                fre_logger.info(f' AFTER assignment: xh_bnds[{i}][0] = {xh_bnds[i][0]}')
+                fre_logger.info(f' AFTER assignment: xh_bnds[{i}][1] = {xh_bnds[i][1]}')
+                fre_logger.info(f'             type(xh_bnds[{i}][1]) = {type(xh_bnds[i][1])}')
 
-        print('\n')
+        fre_logger.info('\n')
         print_data_minmax(yh_bnds[:], "yh_bnds")
         print_data_minmax(xh_bnds[:], "xh_bnds")
-        print('\n')
+        fre_logger.info('\n')
 
 
     # now we set up the cmor module object
@@ -292,11 +292,11 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
     # read experiment configuration file
-    print(f"(rewrite_netcdf_file_var) cmor is opening: json_exp_config = {json_exp_config}")
+    fre_logger.info(f" cmor is opening: json_exp_config = {json_exp_config}")
     cmor.dataset_json(json_exp_config)
 
     # load CMOR table
-    print(f"(rewrite_netcdf_file_var) cmor is loading+setting json_table_config = {json_table_config}")
+    fre_logger.info(f" cmor is loading+setting json_table_config = {json_table_config}")
     loaded_cmor_table_cfg = cmor.load_table(json_table_config)
     cmor.set_table(loaded_cmor_table_cfg)
 
@@ -305,7 +305,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     json_grids_config, loaded_cmor_grids_cfg = None, None
     if process_tripolar_data:
         json_grids_config = str(Path(json_table_config).parent) + '/CMIP6_grids.json'
-        print(f'(rewrite_netcdf_file_var) cmor is loading/opening {json_grids_config}')
+        fre_logger.info(f' cmor is loading/opening {json_grids_config}')
         loaded_cmor_grids_cfg =  cmor.load_table( json_grids_config )
         cmor.set_table(loaded_cmor_grids_cfg)
 
@@ -315,37 +315,37 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     # setup cmor latitude axis if relevant
     cmor_y = None
     if process_tripolar_data:
-        print('(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected y coordinate!!')
+        fre_logger.warning(' calling cmor.axis for a projected y coordinate!!')
         cmor_y = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees")
     elif lat is None :
-        print('(rewrite_netcdf_file_var) WARNING: lat or lat_bnds is None, skipping assigning cmor_y')
+        fre_logger.warning(' lat or lat_bnds is None, skipping assigning cmor_y')
     else:
-        print('(rewrite_netcdf_file_var) assigning cmor_y')
+        fre_logger.info(' assigning cmor_y')
         if lat_bnds is None:
             cmor_y = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N")
         else:
             cmor_y = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N")
-        print('                          DONE assigning cmor_y')
+        fre_logger.info('                          DONE assigning cmor_y')
 
     # setup cmor longitude axis if relevant
     cmor_x = None
     if process_tripolar_data:
-        print('(rewrite_netcdf_file_var) WARNING: calling cmor.axis for a projected x coordinate!!')
+        fre_logger.warning(' calling cmor.axis for a projected x coordinate!!')
         cmor_x = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees")
     elif lon is None :
-        print('(rewrite_netcdf_file_var) WARNING: lon or lon_bnds is None, skipping assigning cmor_x')
+        fre_logger.warning(' lon or lon_bnds is None, skipping assigning cmor_x')
     else:
-        print('(rewrite_netcdf_file_var) assigning cmor_x')
+        fre_logger.info(' assigning cmor_x')
         cmor_x = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E")
         if lon_bnds is None:
             cmor_x = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E")
         else:
             cmor_x = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E")
-        print('                          DONE assigning cmor_x')
+        fre_logger.info('                          DONE assigning cmor_x')
 
     cmor_grid = None
     if process_tripolar_data:
-        print(f'(rewrite_netcdf_file_var) WARNING setting cmor.grid, process_tripolar_data = {process_tripolar_data}')
+        fre_logger.warning(f' setting cmor.grid, process_tripolar_data = {process_tripolar_data}')
         cmor_grid = cmor.grid( axis_ids = [cmor_y, cmor_x],
                                latitude = lat[:], longitude = lon[:],
                                latitude_vertices = lat_bnds[:],
@@ -356,20 +356,20 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
     # setup cmor time axis if relevant
     cmor_time = None
-    print('(rewrite_netcdf_file_var) assigning cmor_time')
+    fre_logger.info(' assigning cmor_time')
     try: #if vert_dim != 'landuse':
-        print(  "(rewrite_netcdf_file_var) Executing cmor.axis('time', \n"
-               f"                         coord_vals = \n{time_coords}, \n"
-               f"                         cell_bounds = time_bnds, units = {time_coord_units})   ")
-        print('(rewrite_netcdf_file_var) assigning cmor_time using time_bnds...')
+        fre_logger.info(  " Executing cmor.axis('time', \n"
+               f"    coord_vals = \n{time_coords}, \n"
+               f"    cell_bounds = time_bnds, units = {time_coord_units})   ")
+        fre_logger.info(' assigning cmor_time using time_bnds...')
         cmor_time = cmor.axis("time", coord_vals = time_coords,
                               cell_bounds = time_bnds, units = time_coord_units)
     except ValueError as exc: #else:
-        print("(rewrite_netcdf_file_var) cmor_time = cmor.axis('time', \n"
+        fre_logger.info(" cmor_time = cmor.axis('time', \n"
                "                          coord_vals = time_coords, units = time_coord_units)")
-        print('(rewrite_netcdf_file_var) assigning cmor_time WITHOUT time_bnds...')
+        fre_logger.info(' assigning cmor_time WITHOUT time_bnds...')
         cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units)
-    print('                          DONE assigning cmor_time')
+    fre_logger.info('                          DONE assigning cmor_time')
 
 
     # other vertical-axis-relevant initializations
@@ -381,10 +381,10 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     # set cmor vertical axis if relevant
     cmor_z = None
     if lev is not None:
-        print('(rewrite_netcdf_file_var) assigning cmor_z')
+        fre_logger.info(' assigning cmor_z')
 
         if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]:
-            print('(rewrite_netcdf_file_var) non-hybrid sigma coordinate case')
+            fre_logger.info(' non-hybrid sigma coordinate case')
             if vert_dim.lower() != "landuse":
                 cmor_vert_dim_name = vert_dim
                 cmor_z = cmor.axis( cmor_vert_dim_name,
@@ -402,8 +402,8 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         elif vert_dim in ["z_l"]:
             lev_bnds = create_lev_bnds( bound_these = lev,
                                          with_these = ds['z_i'] )
-            print('(rewrite_netcdf_file_var) created lev_bnds...')
-            print(f'                          lev_bnds = \n{lev_bnds}')
+            fre_logger.info(' created lev_bnds...')
+            fre_logger.info(f'  lev_bnds = \n{lev_bnds}')
             cmor_z = cmor.axis( 'depth_coord',
                                   coord_vals = lev[:],
                                   units = lev_units,
@@ -449,61 +449,61 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
                                        zfactor_bounds = ds["b_bnds"][:],
                                        units          = ds["b"].units )
 
-            print(f'(rewrite_netcdf_file_var) ierr_ap after calling cmor_zfactor: {ierr_ap}\n'
-                  f'(rewrite_netcdf_file_var) ierr_b after calling cmor_zfactor: {ierr_b}'  )
+            fre_logger.info(f' ierr_ap after calling cmor_zfactor: {ierr_ap}\n'
+                  f' ierr_b after calling cmor_zfactor: {ierr_b}'  )
             axis_ids = []
             if cmor_time is not None:
-                print('(rewrite_netcdf_file_var) appending cmor_time to axis_ids list...')
+                fre_logger.info(' appending cmor_time to axis_ids list...')
                 axis_ids.append(cmor_time)
-                print(f'                          axis_ids now = {axis_ids}')
+                fre_logger.info(f'    axis_ids now = {axis_ids}')
             # might there need to be a conditional check for tripolar ocean data here as well? TODO
             if cmor_y is not None:
-                print('(rewrite_netcdf_file_var) appending cmor_y to axis_ids list...')
+                fre_logger.info(' appending cmor_y to axis_ids list...')
                 axis_ids.append(cmor_y)
-                print(f'                          axis_ids now = {axis_ids}')
+                fre_logger.info(f'    axis_ids now = {axis_ids}')
             if cmor_x is not None:
-                print('(rewrite_netcdf_file_var) appending cmor_x to axis_ids list...')
+                fre_logger.info(' appending cmor_x to axis_ids list...')
                 axis_ids.append(cmor_x)
-                print(f'                          axis_ids now = {axis_ids}')
+                fre_logger.info(f'    axis_ids now = {axis_ids}')
 
             ips = cmor.zfactor( zaxis_id     = cmor_z,
                                 zfactor_name = "ps",
                                 axis_ids     = axis_ids, #[cmor_time, cmor_y, cmor_x],
                                 units        = "Pa" )
             save_ps = True
-        print('                          DONE assigning cmor_z')
+        fre_logger.info('    DONE assigning cmor_z')
 
 
     axes = []
     if cmor_time is not None:
-        print('(rewrite_netcdf_file_var) appending cmor_time to axes list...')
+        fre_logger.info(' appending cmor_time to axes list...')
         axes.append(cmor_time)
-        print(f'                          axes now = {axes}')
+        fre_logger.info(f'    axes now = {axes}')
 
     if cmor_z is not None:
-        print('(rewrite_netcdf_file_var) appending cmor_z to axes list...')
+        fre_logger.info(' appending cmor_z to axes list...')
         axes.append(cmor_z)
-        print(f'                          axes now = {axes}')
+        fre_logger.info(f'    axes now = {axes}')
 
     if process_tripolar_data:
         axes.append(cmor_grid)
     else:
         if cmor_y is not None:
-            print('(rewrite_netcdf_file_var) appending cmor_y to axes list...')
+            fre_logger.info(' appending cmor_y to axes list...')
             axes.append(cmor_y)
-            print(f'                          axes now = {axes}')
+            fre_logger.info(f'    axes now = {axes}')
         if cmor_x is not None:
-            print('(rewrite_netcdf_file_var) appending cmor_x to axes list...')
+            fre_logger.info(' appending cmor_x to axes list...')
             axes.append(cmor_x)
-            print(f'                          axes now = {axes}')
+            fre_logger.info(f'    axes now = {axes}')
 
 
     # read positive/units attribute and create cmor_var
     units = proj_table_vars["variable_entry"] [target_var] ["units"]
-    print(f"(rewrite_netcdf_file_var) units = {units}")
+    fre_logger.info(f" units = {units}")
 
     positive = proj_table_vars["variable_entry"] [target_var] ["positive"]
-    print(f"(rewrite_netcdf_file_var) positive = {positive}")
+    fre_logger.info(f" positive = {positive}")
 
     cmor_var = cmor.variable(target_var, units, axes, positive = positive)
 
@@ -511,17 +511,17 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     cmor.write(cmor_var, var)
     if save_ps:
         if any( [ ips is None, ps is None ] ):
-            print( '(rewrite_netcdf_file_var) WARNING: ps or ips is None!, but save_ps is True!\n'
-                  f'                                   ps = {ps}, ips = {ips}\n'
-                   '                                   skipping ps writing!'    )
+            fre_logger.warning( ' ps or ips is None!, but save_ps is True!\n'
+                  f'    ps = {ps}, ips = {ips}\n'
+                   '    skipping ps writing!'    )
         else:
             cmor.write(ips, ps, store_with = cmor_var)
             cmor.close(ips, file_name = True, preserve = False)
     filename = cmor.close(cmor_var, file_name = True, preserve = False)
-    print(f"(rewrite_netcdf_file_var) returned by cmor.close: filename = {filename}")
+    fre_logger.info(f" returned by cmor.close: filename = {filename}")
     ds.close()
 
-    print('-------------------------- END rewrite_netcdf_file_var call -----\n\n')
+    fre_logger.info('-------------------------- END rewrite_netcdf_file_var call -----\n\n')
     return filename
 
 
@@ -548,16 +548,16 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
         json_table_config: see cmor_run_subtool arg desc
 
     '''
-    print('\n\n-------------------------- START cmorize_target_var_files call -----')
-    print(f"(cmorize_target_var_files) local_var = {local_var} to be used for file-targeting.\n"
-          f"                           target_var = {target_var} to be used for reading the data \n"
-           "                           from the file\n"
-          f"                           outdir = {outdir}")
+    fre_logger.info('\n\n-------------------------- START cmorize_target_var_files call -----')
+    fre_logger.info(f"local_var = {local_var} to be used for file-targeting.\n"
+          f"    target_var = {target_var} to be used for reading the data \n"
+           "    from the file\n"
+          f"    outdir = {outdir}")
 
 
     #determine a tmp dir for working on files.
     tmp_dir = create_tmp_dir( outdir,  json_exp_config) + '/'
-    print(f'(cmorize_target_var_files) will use tmp_dir={tmp_dir}')
+    fre_logger.info(f'will use tmp_dir={tmp_dir}')
 
 
     # loop over sets of dates, each one pointing to a file
@@ -566,45 +566,45 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
 
         # why is nc_fls a filled list/array/object thingy here? see above line
         nc_fls[i] = f"{indir}/{name_of_set}.{iso_datetime}.{local_var}.nc"
-        print(f"(cmorize_target_var_files) input file = {nc_fls[i]}")
+        fre_logger.info(f"input file = {nc_fls[i]}")
         if not Path(nc_fls[i]).exists():
-            print ("(cmorize_target_var_files) input file(s) not found. Moving on.")
+            fre_logger.warning("input file(s) not found. Moving on.")
             continue
 
 
         # create a copy of the input file with local var name into the work directory
         nc_file_work = f"{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc"
 
-        print(f"(cmorize_target_var_files) nc_file_work = {nc_file_work}")
+        fre_logger.info(f"nc_file_work = {nc_file_work}")
         shutil.copy(nc_fls[i], nc_file_work)
 
         # if the ps file exists, we'll copy it to the work directory too
         nc_ps_file      =    nc_fls[i].replace(f'.{local_var}.nc', '.ps.nc')
         nc_ps_file_work = nc_file_work.replace(f'.{local_var}.nc', '.ps.nc')
         if Path(nc_ps_file).exists():
-            print(f"(cmorize_target_var_files) nc_ps_file_work = {nc_ps_file_work}")
+            fre_logger.info(f"nc_ps_file_work = {nc_ps_file_work}")
             shutil.copy(nc_ps_file, nc_ps_file_work)
 
 
         # TODO think of better way to write this kind of conditional data movement...
         # now we have a file in our targets, point CMOR to the configs and the input file(s)
-        make_cmor_write_here = None        #print( Path( tmp_dir     ) )        #print( Path( os.getcwd() ) )
+        make_cmor_write_here = None        #fre_logger.info( Path( tmp_dir     ) )        #fre_logger.info( Path( os.getcwd() ) )
         if Path( tmp_dir ).is_absolute():
-            #print(f'tmp_dir is absolute')
+            #fre_logger.info(f'tmp_dir is absolute')
             make_cmor_write_here = tmp_dir
         elif Path( tmp_dir ).exists(): # relative to where we are
-            #print(f'tmp_dir is relative to CWD!')
+            #fre_logger.info(f'tmp_dir is relative to CWD!')
             make_cmor_write_here = os.getcwd() + '/'+tmp_dir # unavoidable, cmor module FORCES write to CWD
         assert make_cmor_write_here is not None
 
         gotta_go_back_here = os.getcwd()
         try:
-            print(f"(cmorize_target_var_files) WARNING changing directory to: \n {make_cmor_write_here}" )
+            fre_logger.warning(f" changing directory to: \n {make_cmor_write_here}" )
             os.chdir( make_cmor_write_here )
         except:
-            raise OSError(f'could not chdir to {make_cmor_write_here}')
+            raise OSError(f'(cmorize_target_var_files) could not chdir to {make_cmor_write_here}')
 
-        print ("(cmorize_target_var_files) calling rewrite_netcdf_file_var")
+        fre_logger.info("calling rewrite_netcdf_file_var")
         try:
             local_file_name = rewrite_netcdf_file_var( proj_table_vars      ,
                                                        local_var            ,
@@ -617,28 +617,28 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
                             f'                           {exc}\n'
                             '                           exiting and executing finally block.')
         finally: # should always execute, errors or not!
-            print(f'(cmorize_target_var_files) WARNING changing directory to: \n      {gotta_go_back_here}')
+            fre_logger.warning(f'changing directory to: \n      {gotta_go_back_here}')
             os.chdir( gotta_go_back_here )
 
 
         # now that CMOR has rewritten things... we can take our post-rewriting actions
         # the final output filename will be...
-        print(f'(cmorize_target_var_files) local_file_name = {local_file_name}')
+        fre_logger.info(f'local_file_name = {local_file_name}')
         filename  = f"{outdir}/{local_file_name}"
-        print(f"(cmorize_target_var_files) filename = {filename}")
+        fre_logger.info(f"filename = {filename}")
 
         # the final output file directory will be...
         filedir = Path(filename).parent
-        print(f"(cmorize_target_var_files) filedir = {filedir}")
+        fre_logger.info(f"filedir = {filedir}")
         try:
-            print(f'(cmorize_target_var_files) attempting to create filedir={filedir}')
+            fre_logger.info(f'attempting to create filedir={filedir}')
             os.makedirs(filedir)
         except FileExistsError:
-            print(f'(cmorize_target_var_files) WARNING: directory {filedir} already exists!')
+            fre_logger.warning(f' directory {filedir} already exists!')
 
         # hmm.... this is making issues for pytest
         mv_cmd = f"mv {tmp_dir}/{local_file_name} {filedir}"
-        print(f"(cmorize_target_var_files) moving files...\n {mv_cmd}")
+        fre_logger.info(f"moving files...\n {mv_cmd}")
         subprocess.run(mv_cmd, shell = True, check = True)
 
         # ------ refactor this into function? #TODO
@@ -646,11 +646,11 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
         filename_no_nc = filename[:filename.rfind(".nc")]
         chunk_str = filename_no_nc[-6:]
         if not chunk_str.isdigit():
-            print(f'(cmorize_target_var_files) WARNING: chunk_str is not a digit: '
+            fre_logger.warning(f' chunk_str is not a digit: '
                   f'chunk_str = {chunk_str}')
             filename_corr = "{filename[:filename.rfind('.nc')]}_{iso_datetime}.nc"
             mv_cmd = f"mv {filename} {filename_corr}"
-            print(f"(cmorize_target_var_files) moving files, strange chunkstr logic...\n {mv_cmd}")
+            fre_logger.warning(f"moving files, strange chunkstr logic...\n {mv_cmd}")
             subprocess.run(mv_cmd, shell = True, check = True)
         # ------ end refactor this into function?
 
@@ -662,8 +662,8 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
             Path(nc_ps_file_work).unlink()
 
         if DEBUG_MODE_RUN_ONE:
-            print('WARNING: DEBUG_MODE_RUN_ONE is True!!!!')
-            print('WARNING: done processing one file!!!')
+            fre_logger.warning('DEBUG_MODE_RUN_ONE is True!!!!')
+            fre_logger.warning('done processing one file!!!')
             break
 
 
@@ -703,7 +703,7 @@ def cmor_run_subtool( indir = None,
 
     ## open CMOR table config file
     fre_logger.info( 'loading json_table_config = \n'
-           f'    {json_table_config}\n'                     )
+                     f'    {json_table_config}\n'                     )
     try:
         with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
             proj_table_vars = json.load(table_config_file)
@@ -717,8 +717,8 @@ def cmor_run_subtool( indir = None,
     json_table_config= str( Path(json_table_config).resolve() )
 
     # open input variable list
-    print('(cmor_run_subtool) loading json_var_list = \n '
-          '                               {var_list_file}\n')
+    fre_logger.info('loading json_var_list = \n '
+                    f'    {json_var_list}\n')
     try:
         with open( json_var_list, "r", encoding = "utf-8"  ) as var_list_file:
             var_list = json.load( var_list_file )
@@ -741,46 +741,45 @@ def cmor_run_subtool( indir = None,
 
         # if its not in the table configurations variable_entry list, skip
         if var_list[local_var] not in proj_table_vars["variable_entry"]:
-            print(f"(cmor_run_subtool) WARNING: skipping local_var  = {local_var} /\n"
-                  f"                                     target_var = {var_list[local_var]}\n"
-                   "                        ... target_var not found in CMOR variable group")
+            fre_logger.warning(f"skipping local_var  = {local_var} /\n"
+                  f"... target_var = {var_list[local_var]}\n"
+                   "... target_var not found in CMOR variable group")
             continue
 
         if all( [ opt_var_name is not None,
                   local_var != opt_var_name ] ):
-            print(f'(cmor_run_subtool) WARNING: skipping local_var = {local_var} as it is not equal\n'
-                   '                            to the opt_var_name argument.')
+            fre_logger.warning(f'skipping local_var = {local_var} as it is not equal\n'
+                   '... to the opt_var_name argument.')
             continue
-        print('\n')
 
         # it is in there, get the name of the data inside the netcdf file.
         target_var = var_list[local_var] # often equiv to local_var but not necessarily.
         if local_var != target_var:
-            print(f'(cmor_run_subtool) WARNING: local_var == {local_var} \n'
-                  f'                            != {target_var} == target_var\n'
-                  f'                            i am expecting {local_var} to be in the filename, and i expect the variable\n'
-                  f'                            in that file to be {target_var}')
+            fre_logger.warning(f'local_var == {local_var} \n'
+                  f'    != {target_var} == target_var\n'
+                  f'    i am expecting {local_var} to be in the filename, and i expect the variable\n'
+                  f'    in that file to be {target_var}')
 
 
         # examine input directory to obtain a list of input file targets
         var_filenames = []
         get_var_filenames(indir, var_filenames, local_var)
-        print(f"(cmor_run_subtool) found filenames = \n {var_filenames}\n")
+        fre_logger.info(f" found filenames = \n {var_filenames}\n")
 
         # examine input files to obtain target date ranges
         iso_datetime_arr = []
         get_iso_datetimes(var_filenames, iso_datetime_arr)
-        print(f"(cmor_run_subtool) found iso datetimes = \n {iso_datetime_arr}\n")
+        fre_logger.info(f" found iso datetimes = \n {iso_datetime_arr}\n")
 
         # name_of_set == component label...
         # which is not relevant for CMOR/CMIP... or is it?
         name_of_set = var_filenames[0].split(".")[0]
-        print(f"(cmor_run_subtool) setting name_of_set = {name_of_set}")
+        fre_logger.info(f" setting name_of_set = {name_of_set}")
 
 
 
-        print(f'(cmor_run_subtool) ..............beginning CMORization for {local_var}/\n'
-              f'                                                 {target_var}..........')
+        fre_logger.info(f'........beginning CMORization for {local_var}/\n'
+                        f'                        {target_var}..........')
         cmorize_target_var_files(
             indir, target_var, local_var, iso_datetime_arr, # OK
             name_of_set, json_exp_config,
@@ -789,6 +788,6 @@ def cmor_run_subtool( indir = None,
         )
 
         if DEBUG_MODE_RUN_ONE:
-            print('WARNING: DEBUG_MODE_RUN_ONE is True. breaking var_list loop')
+            fre_logger.warning('DEBUG_MODE_RUN_ONE is True. breaking var_list loop')
             break
     return 0
diff --git a/fre/fre.py b/fre/fre.py
index 943557a2..b935f82c 100644
--- a/fre/fre.py
+++ b/fre/fre.py
@@ -56,6 +56,8 @@ def fre(verbose):
     ''' entry point function to subgroup functions '''
     if verbose:
         logging.basicConfig(level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.WARN)
 
 if __name__ == '__main__':
     fre()
diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py
index e2f64d6a..d0eedb5f 100644
--- a/fre/pp/checkout_script.py
+++ b/fre/pp/checkout_script.py
@@ -9,7 +9,7 @@
 import sys
 import subprocess
 
-from fre.fre import version as fre_ver
+from ..fre import version as fre_ver
 
 FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git'
 

From 1e5a69f2fe9463d5e614c2542b6d8869199edd82 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 13:35:47 -0500
Subject: [PATCH 04/23] loggers... loggers... EVERYWHERE!!!!

---
 fre/cmor/cmor_finder.py  | 47 ++++++++++++-----------
 fre/cmor/cmor_helpers.py | 82 +++++++++++++++++++++-------------------
 2 files changed, 69 insertions(+), 60 deletions(-)

diff --git a/fre/cmor/cmor_finder.py b/fre/cmor/cmor_finder.py
index c86c5635..aa231f25 100644
--- a/fre/cmor/cmor_finder.py
+++ b/fre/cmor/cmor_finder.py
@@ -12,6 +12,9 @@
 
 import glob
 import json
+import logging
+fre_logger = logging.getLogger(__name__)
+
 from pathlib import Path
 
 DO_NOT_PRINT_LIST=[ 'comment',
@@ -29,35 +32,35 @@ def print_var_content( table_config_file = None, var_name = None):
     try:
         var_content = proj_table_vars["variable_entry"].get(var_name)
     except:
-        #print(f'(cmor_find_subtool) WARNING no "variable_entry" key. for {json_table_config}.'
-        #       '                    not the right json file probably. moving on!')
+        fre_logger.warning(f' no "variable_entry" key. for {json_table_config}.'
+                         '     not the right json file probably. moving on!')
         return
 
     if var_content is None:
-        #print(f'(cmor_find_subtool) variable {var_name} not found in {Path(json_table_config).name}, moving on!')
+        fre_logger.warning(f' variable {var_name} not found in {Path(json_table_config).name}, moving on!')
         return
 
     table_name = None
     try:
-        #print(f'(print_var_content) trying to get table_name from proj_table_vars...')
-        #print(f'                    table header is {proj_table_vars["Header"]}')
+        #fre_logger.info(f' trying to get table_name from proj_table_vars...')
+        #fre_logger.info(f'    table header is {proj_table_vars["Header"]}')
         table_name = proj_table_vars["Header"].get('table_id').split(' ')[1]
-        #print(f'                    table_name = {table_name}')
+        #fre_logger.info(f'    table_name = {table_name}')
     except:
-        print('print_var_content) WARNING couldnt get header and table_name field')
+        fre_logger.warning(' couldnt get header and table_name field')
         pass
 
     if table_name is not None:
-        print(f'(print_var_content) found {var_name} data in table {table_name}!')
+        fre_logger.info(f' found {var_name} data in table {table_name}!')
     else:
-        print(f'(print_var_content) found {var_name} data in table, but not its table_name!')
+        fre_logger.info(f' found {var_name} data in table, but not its table_name!')
 
-    print(f'              variable key: {var_name}')
+    fre_logger.info(f'    variable key: {var_name}')
     for content in var_content:
         if content in DO_NOT_PRINT_LIST:
             continue
-        print(f'              {content}: {var_content[content]}')
-    print('\n')
+        fre_logger.info(f'    {content}: {var_content[content]}')
+    fre_logger.info('\n')
 
     return
 
@@ -67,14 +70,14 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v
     out to screen, intended largely as a helper tool for cli users.
     '''
     if not Path(json_table_config_dir).exists():
-        raise OSError(f'(cmor_find_subtool) ERROR directory {json_table_config_dir} does not exist! exit.')
+        raise OSError(f' ERROR directory {json_table_config_dir} does not exist! exit.')
 
-    print(f'(cmor_find_subtool) attempting to find and open files in dir: \n {json_table_config_dir} ')
+    fre_logger.info(f' attempting to find and open files in dir: \n {json_table_config_dir} ')
     json_table_configs=glob.glob(f'{json_table_config_dir}/CMIP6_*.json')
     if json_table_configs is None:
         raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.')
     else:
-        print('(cmor_find_subtool) found content in json_table_config_dir')
+        fre_logger.info(' found content in json_table_config_dir')
 
     var_list = None
     if json_var_list is not None:
@@ -82,25 +85,25 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v
             var_list=json.load(var_list_file)
 
     if opt_var_name is None and var_list is None:
-        raise ValueError('(cmor_find_subtool) ERROR: no opt_var_name given but also no content in variable list!!! exit!')
+        raise ValueError(' ERROR: no opt_var_name given but also no content in variable list!!! exit!')
 
     if opt_var_name is not None:
-        print('(cmor_find_subtool) opt_var_name is not None: looking for only ONE variables worth of info!')
+        fre_logger.info(' opt_var_name is not None: looking for only ONE variables worth of info!')
         for json_table_config in json_table_configs:
-            #print(f'(cmor_find_subtool) attempting to open {json_table_config}')
+            #fre_logger.info(f' attempting to open {json_table_config}')
             with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
                 print_var_content(table_config_file, opt_var_name)
 
     elif var_list is not None:
-        print('(cmor_find_subtool) opt_var_name is None, and var_list is not None, looking for many variables worth of info!')
+        fre_logger.info(' opt_var_name is None, and var_list is not None, looking for many variables worth of info!')
         for var in var_list:
             for json_table_config in json_table_configs:
-                #print(f'(cmor_find_subtool) attempting to open {json_table_config}')
+                #fre_logger.info(f' attempting to open {json_table_config}')
                 with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
-                    #print(f'    var = {var}, var_list[{var}]={var_list[var]}')
+                    #fre_logger.info(f'    var = {var}, var_list[{var}]={var_list[var]}')
                     print_var_content(table_config_file, str(var_list[var]))
     else:
-        print('(FATAL) this line should be unreachable!!!')
+        fre_logger.error(' this line should be unreachable!!!')
         assert False
 
     return
diff --git a/fre/cmor/cmor_helpers.py b/fre/cmor/cmor_helpers.py
index de961728..4b55a21a 100644
--- a/fre/cmor/cmor_helpers.py
+++ b/fre/cmor/cmor_helpers.py
@@ -1,24 +1,31 @@
-import os
-import json
+
+
+
 import glob
-from pathlib import Path
+import json
+
+import logging
+fre_logger = logging.getLogger(__name__)
+
 import numpy as np
+import os
+from pathlib import Path
 
 
 def print_data_minmax(ds_variable = None, desc = None):
     '''
     prints the min/max of numpy.ma.core.MaskedArray (ds_variable) and the name/description (desc) of the data
     '''
-    print('----------------------------------------------------------------------------------------------------------')
+    fre_logger.info('----------------------------------------------------------------------------------------------------------')
     try:
-        print(f'(print_data_minmax) info for \n'
+        fre_logger.info(f' info for \n'
               f'                     desc = {desc} \n {type(ds_variable)}')
-        #print(f'(print_data_minmax) {ds_variable.data.min()} < {desc} < {ds_variable.data.max()}')
-        print(f'(print_data_minmax) {ds_variable.min()} < {desc} < {ds_variable.max()}')
+        #fre_logger.info(f' {ds_variable.data.min()} < {desc} < {ds_variable.data.max()}')
+        fre_logger.info(f' {ds_variable.min()} < {desc} < {ds_variable.max()}')
     except:
-        print(f'(print_data_minmax) could not print min/max entries for desc = {desc}')
+        fre_logger.warning(f' could not print min/max entries for desc = {desc}')
         pass
-    print('----------------------------------------------------------------------------------------------------------')
+    fre_logger.info('----------------------------------------------------------------------------------------------------------')
     return
 
 
@@ -31,43 +38,42 @@ def from_dis_gimme_dis(from_dis, gimme_dis):
     try:
         return from_dis[gimme_dis][:].copy()
     except Exception as exc:
-        print(f'(from_dis_gimme_dis) WARNING I am sorry, I could not not give you this: {gimme_dis}\n'
-              #f'                                                             from this: {from_dis} \n'
-              #f'             exc = {exc}\n'
+        fre_logger.warning(f' I am sorry, I could not not give you this: {gimme_dis}\n'
               '            returning None!\n'                           )
         return None
 
 def find_statics_file(bronx_file_path):
     bronx_file_path_elem = bronx_file_path.split('/')
     num_elem = len(bronx_file_path_elem)
-    print(f'(find_statics_file) bronx_file_path_elem = \n{bronx_file_path_elem}\n')
+    fre_logger.info(f' bronx_file_path_elem = \n{bronx_file_path_elem}\n')
     while bronx_file_path_elem[num_elem-2] != 'pp':
         bronx_file_path_elem.pop()
         num_elem = num_elem-1
-        #print(bronx_file_path_elem)
+        #fre_logger.info(bronx_file_path_elem)
     statics_path = '/'.join(bronx_file_path_elem)
     statics_file = glob.glob(statics_path+'/*static*.nc')[0]
     if Path(statics_file).exists():
         return statics_file
     else:
+        fre_logger.warning(' could not find the statics file! returning None')
         return None
 
 
 def create_lev_bnds(bound_these = None, with_these = None):
     the_bnds = None
     assert len(with_these) == len(bound_these) + 1
-    print( '(create_lev_bnds) bound_these is... ')
-    print(f'                  bound_these = \n{bound_these}')
-    print( '(create_lev_bnds) with_these is... ')
-    print(f'                  with_these = \n{with_these}')
+    fre_logger.info( ' bound_these is... ')
+    fre_logger.info(f' bound_these = \n{bound_these}')
+    fre_logger.info( ' with_these is... ')
+    fre_logger.info(f' with_these = \n{with_these}')
 
 
     the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2)
     for i in range(0,len(bound_these)):
         the_bnds[i][0] = with_these[i]
         the_bnds[i][1] = with_these[i+1]
-    print( '(create_lev_bnds) the_bnds is... ')
-    print(f'                  the_bnds = \n{the_bnds}')
+    fre_logger.info( ' the_bnds is... ')
+    fre_logger.info(f' the_bnds = \n{the_bnds}')
     return the_bnds
 
 def get_var_filenames(indir, var_filenames = None, local_var = None):
@@ -82,13 +88,13 @@ def get_var_filenames(indir, var_filenames = None, local_var = None):
     if var_filenames is None:
         var_filenames = []
     filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc'
-    print(f'(get_var_filenames) filename_pattern = {filename_pattern}\n')
-    print(f'(get_var_filenames) indir = {indir}\n')
+    fre_logger.info(f' filename_pattern = {filename_pattern}\n')
+    fre_logger.info(f' indir = {indir}\n')
     var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}')
-    #print(f'(get_var_filenames) var_filenames_all = {var_filenames_all}')
+    #fre_logger.info(f' var_filenames_all = {var_filenames_all}')
     for var_file in var_filenames_all:
         var_filenames.append( Path(var_file).name )
-    #print(f"(get_var_filenames) var_filenames = {var_filenames}")
+    #fre_logger.info(f" var_filenames = {var_filenames}")
     if len(var_filenames) < 1:
         raise ValueError(f'target directory had no files with .nc ending. indir =\n {indir}')
     var_filenames.sort()
@@ -111,9 +117,9 @@ def get_iso_datetimes(var_filenames, iso_datetime_arr = None):
             iso_datetime_arr.append(
                 filename.split(".")[1] )
     iso_datetime_arr.sort()
-    #print(f"(get_iso_datetimes) Available dates: {iso_datetime_arr}")
+    #fre_logger.info(f" Available dates: {iso_datetime_arr}")
     if len(iso_datetime_arr) < 1:
-        raise ValueError('(get_iso_datetimes) ERROR: iso_datetime_arr has length 0!')
+        raise ValueError(' ERROR: iso_datetime_arr has length 0!')
 
 
 def check_dataset_for_ocean_grid(ds):
@@ -123,9 +129,9 @@ def check_dataset_for_ocean_grid(ds):
         ds: netCDF4.Dataset object containing variables with associated dimensional information.
     '''
     if "xh" in list(ds.variables.keys()):
-        print("\n----------------------------------------------------------------------------------\n"
-              "(check_dataset_for_ocean_grid) WARNING: 'xh' found in var_list: ocean grid req'd\n"
-              "                                        sometimes i don't cmorize right! check me!\n"
+        fre_logger.warning("\n----------------------------------------------------------------------------------\n"
+              " 'xh' found in var_list: ocean grid req'd\n"
+              "     sometimes i don't cmorize right! check me!\n"
               "----------------------------------------------------------------------------------\n"
         )
         return True
@@ -145,7 +151,7 @@ def get_vertical_dimension(ds, target_var):
         if name != target_var: # not the var we are looking for? move on.
             continue
         dims = variable.dimensions
-        for dim in dims: #print(f'(get_vertical_dimension) dim={dim}')
+        for dim in dims: #fre_logger.info(f'(get_vertical_dimension) dim={dim}')
 
             # check for special case
             if dim.lower() == 'landuse': # aux coordinate, so has no axis property
@@ -174,8 +180,8 @@ def create_tmp_dir(outdir, json_exp_config = None):
             try:
                 outdir_from_exp_config = json.load(table_config_file)["outpath"]
             except:
-                print( '(create_tmp_dir) WARNING could not read outdir from json_exp_config.'
-                       '                 the cmor module will throw a toothless warning'     )
+                fre_logger.warning( ' could not read outdir from json_exp_config.'
+                       '   the cmor module will throw a toothless warning'     )
 
     # assign an appropriate temporary working directory
     tmp_dir = None
@@ -183,24 +189,24 @@ def create_tmp_dir(outdir, json_exp_config = None):
               outdir.find("/work") != -1,
               outdir.find("/net" ) != -1 ] ):
         tmp_dir = str( Path("{outdir}/").resolve() )
-        print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {tmp_dir} )')
+        fre_logger.info(f' using /local /work /net ( tmp_dir = {tmp_dir} )')
     else:
         tmp_dir = str( Path(f"{outdir}/tmp/").resolve() )
-        print(f'(create_tmp_dir) NOT using /local /work /net ( tmp_dir = {tmp_dir} )')
+        fre_logger.info(f' NOT using /local /work /net ( tmp_dir = {tmp_dir} )')
 
     # once we know where the tmp_dir should be, create it
     try:
         os.makedirs(tmp_dir, exist_ok = True)
         # and if we need to additionally create outdir_from_exp_config... try doing that too
         if outdir_from_exp_config is not None:
-            print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ')
+            fre_logger.info(f' attempting to create {outdir_from_exp_config} dir in tmp_dir targ')
             try:
                 os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok = True)
             except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too.
-                print(f'(create_tmp_dir) attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work')
-                print( '                 .... oh well! it was ust to try to avoid a warning anyways.... moving on')
+                fre_logger.info(f' attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work')
+                fre_logger.info( '                 .... oh well! it was ust to try to avoid a warning anyways.... moving on')
                 pass
     except Exception as exc:
-        raise OSError(f'(create_tmp_dir) problem creating tmp output directory {tmp_dir}. stop.') from exc
+        raise OSError(f' problem creating tmp output directory {tmp_dir}. stop.') from exc
 
     return tmp_dir

From ed23206a5d2b5c38d2ed79e030cc6b23ac437d11 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 14:32:11 -0500
Subject: [PATCH 05/23] put new log settings in pytest.ini. set format of
 logging in fre.py and give new log_file argument. remove needlless space at
 the beginning of most current logging statements (that were print
 statements).

---
 fre/cmor/cmor_finder.py  |  30 ++++-----
 fre/cmor/cmor_helpers.py |  55 ++++++++-------
 fre/cmor/cmor_mixer.py   | 140 +++++++++++++++++++--------------------
 fre/fre.py               |  14 +++-
 fre/pytest.ini           |   5 ++
 5 files changed, 127 insertions(+), 117 deletions(-)

diff --git a/fre/cmor/cmor_finder.py b/fre/cmor/cmor_finder.py
index aa231f25..3b4b5cfc 100644
--- a/fre/cmor/cmor_finder.py
+++ b/fre/cmor/cmor_finder.py
@@ -32,28 +32,28 @@ def print_var_content( table_config_file = None, var_name = None):
     try:
         var_content = proj_table_vars["variable_entry"].get(var_name)
     except:
-        fre_logger.warning(f' no "variable_entry" key. for {json_table_config}.'
+        fre_logger.warning(f'no "variable_entry" key. for {json_table_config}.'
                          '     not the right json file probably. moving on!')
         return
 
     if var_content is None:
-        fre_logger.warning(f' variable {var_name} not found in {Path(json_table_config).name}, moving on!')
+        fre_logger.warning(f'variable {var_name} not found in {Path(json_table_config).name}, moving on!')
         return
 
     table_name = None
     try:
-        #fre_logger.info(f' trying to get table_name from proj_table_vars...')
+        #fre_logger.info(f'trying to get table_name from proj_table_vars...')
         #fre_logger.info(f'    table header is {proj_table_vars["Header"]}')
         table_name = proj_table_vars["Header"].get('table_id').split(' ')[1]
         #fre_logger.info(f'    table_name = {table_name}')
     except:
-        fre_logger.warning(' couldnt get header and table_name field')
+        fre_logger.warning('couldnt get header and table_name field')
         pass
 
     if table_name is not None:
-        fre_logger.info(f' found {var_name} data in table {table_name}!')
+        fre_logger.info(f'found {var_name} data in table {table_name}!')
     else:
-        fre_logger.info(f' found {var_name} data in table, but not its table_name!')
+        fre_logger.info(f'found {var_name} data in table, but not its table_name!')
 
     fre_logger.info(f'    variable key: {var_name}')
     for content in var_content:
@@ -70,14 +70,14 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v
     out to screen, intended largely as a helper tool for cli users.
     '''
     if not Path(json_table_config_dir).exists():
-        raise OSError(f' ERROR directory {json_table_config_dir} does not exist! exit.')
+        raise OSError(f'ERROR directory {json_table_config_dir} does not exist! exit.')
 
-    fre_logger.info(f' attempting to find and open files in dir: \n {json_table_config_dir} ')
+    fre_logger.info(f'attempting to find and open files in dir: \n {json_table_config_dir} ')
     json_table_configs=glob.glob(f'{json_table_config_dir}/CMIP6_*.json')
     if json_table_configs is None:
         raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.')
     else:
-        fre_logger.info(' found content in json_table_config_dir')
+        fre_logger.info('found content in json_table_config_dir')
 
     var_list = None
     if json_var_list is not None:
@@ -85,25 +85,25 @@ def cmor_find_subtool( json_var_list = None, json_table_config_dir = None, opt_v
             var_list=json.load(var_list_file)
 
     if opt_var_name is None and var_list is None:
-        raise ValueError(' ERROR: no opt_var_name given but also no content in variable list!!! exit!')
+        raise ValueError('ERROR: no opt_var_name given but also no content in variable list!!! exit!')
 
     if opt_var_name is not None:
-        fre_logger.info(' opt_var_name is not None: looking for only ONE variables worth of info!')
+        fre_logger.info('opt_var_name is not None: looking for only ONE variables worth of info!')
         for json_table_config in json_table_configs:
-            #fre_logger.info(f' attempting to open {json_table_config}')
+            #fre_logger.info(f'attempting to open {json_table_config}')
             with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
                 print_var_content(table_config_file, opt_var_name)
 
     elif var_list is not None:
-        fre_logger.info(' opt_var_name is None, and var_list is not None, looking for many variables worth of info!')
+        fre_logger.info('opt_var_name is None, and var_list is not None, looking for many variables worth of info!')
         for var in var_list:
             for json_table_config in json_table_configs:
-                #fre_logger.info(f' attempting to open {json_table_config}')
+                #fre_logger.info(f'attempting to open {json_table_config}')
                 with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
                     #fre_logger.info(f'    var = {var}, var_list[{var}]={var_list[var]}')
                     print_var_content(table_config_file, str(var_list[var]))
     else:
-        fre_logger.error(' this line should be unreachable!!!')
+        fre_logger.error('this line should be unreachable!!!')
         assert False
 
     return
diff --git a/fre/cmor/cmor_helpers.py b/fre/cmor/cmor_helpers.py
index 4b55a21a..4a13c8d8 100644
--- a/fre/cmor/cmor_helpers.py
+++ b/fre/cmor/cmor_helpers.py
@@ -1,6 +1,3 @@
-
-
-
 import glob
 import json
 
@@ -18,12 +15,12 @@ def print_data_minmax(ds_variable = None, desc = None):
     '''
     fre_logger.info('----------------------------------------------------------------------------------------------------------')
     try:
-        fre_logger.info(f' info for \n'
-              f'                     desc = {desc} \n {type(ds_variable)}')
-        #fre_logger.info(f' {ds_variable.data.min()} < {desc} < {ds_variable.data.max()}')
-        fre_logger.info(f' {ds_variable.min()} < {desc} < {ds_variable.max()}')
+        fre_logger.info(f'info for \n'
+              f'                    desc = {desc} \n {type(ds_variable)}')
+        #fre_logger.info(f'{ds_variable.data.min()} < {desc} < {ds_variable.data.max()}')
+        fre_logger.info(f'{ds_variable.min()} < {desc} < {ds_variable.max()}')
     except:
-        fre_logger.warning(f' could not print min/max entries for desc = {desc}')
+        fre_logger.warning(f'could not print min/max entries for desc = {desc}')
         pass
     fre_logger.info('----------------------------------------------------------------------------------------------------------')
     return
@@ -38,14 +35,14 @@ def from_dis_gimme_dis(from_dis, gimme_dis):
     try:
         return from_dis[gimme_dis][:].copy()
     except Exception as exc:
-        fre_logger.warning(f' I am sorry, I could not not give you this: {gimme_dis}\n'
-              '            returning None!\n'                           )
+        fre_logger.warning(f'I am sorry, I could not not give you this: {gimme_dis}\n'
+              '           returning None!\n'                           )
         return None
 
 def find_statics_file(bronx_file_path):
     bronx_file_path_elem = bronx_file_path.split('/')
     num_elem = len(bronx_file_path_elem)
-    fre_logger.info(f' bronx_file_path_elem = \n{bronx_file_path_elem}\n')
+    fre_logger.info(f'bronx_file_path_elem = \n{bronx_file_path_elem}\n')
     while bronx_file_path_elem[num_elem-2] != 'pp':
         bronx_file_path_elem.pop()
         num_elem = num_elem-1
@@ -55,25 +52,25 @@ def find_statics_file(bronx_file_path):
     if Path(statics_file).exists():
         return statics_file
     else:
-        fre_logger.warning(' could not find the statics file! returning None')
+        fre_logger.warning('could not find the statics file! returning None')
         return None
 
 
 def create_lev_bnds(bound_these = None, with_these = None):
     the_bnds = None
     assert len(with_these) == len(bound_these) + 1
-    fre_logger.info( ' bound_these is... ')
-    fre_logger.info(f' bound_these = \n{bound_these}')
-    fre_logger.info( ' with_these is... ')
-    fre_logger.info(f' with_these = \n{with_these}')
+    fre_logger.info( 'bound_these is... ')
+    fre_logger.info(f'bound_these = \n{bound_these}')
+    fre_logger.info( 'with_these is... ')
+    fre_logger.info(f'with_these = \n{with_these}')
 
 
     the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these),2)
     for i in range(0,len(bound_these)):
         the_bnds[i][0] = with_these[i]
         the_bnds[i][1] = with_these[i+1]
-    fre_logger.info( ' the_bnds is... ')
-    fre_logger.info(f' the_bnds = \n{the_bnds}')
+    fre_logger.info( 'the_bnds is... ')
+    fre_logger.info(f'the_bnds = \n{the_bnds}')
     return the_bnds
 
 def get_var_filenames(indir, var_filenames = None, local_var = None):
@@ -88,10 +85,10 @@ def get_var_filenames(indir, var_filenames = None, local_var = None):
     if var_filenames is None:
         var_filenames = []
     filename_pattern = '.nc' if local_var is None else f'.{local_var}.nc'
-    fre_logger.info(f' filename_pattern = {filename_pattern}\n')
-    fre_logger.info(f' indir = {indir}\n')
+    fre_logger.info(f'filename_pattern = {filename_pattern}\n')
+    fre_logger.info(f'indir = {indir}\n')
     var_filenames_all = glob.glob(f'{indir}/*{filename_pattern}')
-    #fre_logger.info(f' var_filenames_all = {var_filenames_all}')
+    #fre_logger.info(f'var_filenames_all = {var_filenames_all}')
     for var_file in var_filenames_all:
         var_filenames.append( Path(var_file).name )
     #fre_logger.info(f" var_filenames = {var_filenames}")
@@ -119,7 +116,7 @@ def get_iso_datetimes(var_filenames, iso_datetime_arr = None):
     iso_datetime_arr.sort()
     #fre_logger.info(f" Available dates: {iso_datetime_arr}")
     if len(iso_datetime_arr) < 1:
-        raise ValueError(' ERROR: iso_datetime_arr has length 0!')
+        raise ValueError('ERROR: iso_datetime_arr has length 0!')
 
 
 def check_dataset_for_ocean_grid(ds):
@@ -180,7 +177,7 @@ def create_tmp_dir(outdir, json_exp_config = None):
             try:
                 outdir_from_exp_config = json.load(table_config_file)["outpath"]
             except:
-                fre_logger.warning( ' could not read outdir from json_exp_config.'
+                fre_logger.warning( 'could not read outdir from json_exp_config.'
                        '   the cmor module will throw a toothless warning'     )
 
     # assign an appropriate temporary working directory
@@ -189,24 +186,24 @@ def create_tmp_dir(outdir, json_exp_config = None):
               outdir.find("/work") != -1,
               outdir.find("/net" ) != -1 ] ):
         tmp_dir = str( Path("{outdir}/").resolve() )
-        fre_logger.info(f' using /local /work /net ( tmp_dir = {tmp_dir} )')
+        fre_logger.info(f'using /local /work /net ( tmp_dir = {tmp_dir} )')
     else:
         tmp_dir = str( Path(f"{outdir}/tmp/").resolve() )
-        fre_logger.info(f' NOT using /local /work /net ( tmp_dir = {tmp_dir} )')
+        fre_logger.info(f'NOT using /local /work /net ( tmp_dir = {tmp_dir} )')
 
     # once we know where the tmp_dir should be, create it
     try:
         os.makedirs(tmp_dir, exist_ok = True)
         # and if we need to additionally create outdir_from_exp_config... try doing that too
         if outdir_from_exp_config is not None:
-            fre_logger.info(f' attempting to create {outdir_from_exp_config} dir in tmp_dir targ')
+            fre_logger.info(f'attempting to create {outdir_from_exp_config} dir in tmp_dir targ')
             try:
                 os.makedirs(tmp_dir+'/'+outdir_from_exp_config, exist_ok = True)
             except: # ... but don't error out for lack of success here, not worth it. cmor can do the lift too.
-                fre_logger.info(f' attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work')
-                fre_logger.info( '                 .... oh well! it was ust to try to avoid a warning anyways.... moving on')
+                fre_logger.info(f'attempting to create {outdir_from_exp_config} dir in tmp_dir targ did not work')
+                fre_logger.info( '                .... oh well! it was ust to try to avoid a warning anyways.... moving on')
                 pass
     except Exception as exc:
-        raise OSError(f' problem creating tmp output directory {tmp_dir}. stop.') from exc
+        raise OSError(f'problem creating tmp output directory {tmp_dir}. stop.') from exc
 
     return tmp_dir
diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py
index f0cc301c..95377200 100755
--- a/fre/cmor/cmor_mixer.py
+++ b/fre/cmor/cmor_mixer.py
@@ -57,41 +57,41 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
     # ocean grids are not implemented yet.
-    fre_logger.info( ' checking input netcdf file for oceangrid condition')
+    fre_logger.info( 'checking input netcdf file for oceangrid condition')
     uses_ocean_grid = check_dataset_for_ocean_grid(ds)
     if uses_ocean_grid:
-        fre_logger.info(' OH BOY you have a file on the native tripolar grid...\n'
+        fre_logger.info('OH BOY you have a file on the native tripolar grid...\n'
               '... this is gonna be fun!' )
 
     # try to read what coordinate(s) we're going to be expecting for the variable
     expected_mip_coord_dims = None
     try:
         expected_mip_coord_dims = proj_table_vars["variable_entry"] [target_var] ["dimensions"]
-        fre_logger.info( ' i am hoping to find data for the following coordinate dimensions:\n'
+        fre_logger.info( 'i am hoping to find data for the following coordinate dimensions:\n'
               f'    expected_mip_coord_dims = {expected_mip_coord_dims}\n' )
     except Exception as exc:
-        fre_logger.warning(f' could not get expected coordinate dimensions for {target_var}. '
+        fre_logger.warning(f'could not get expected coordinate dimensions for {target_var}. '
                '    in proj_table_vars file {json_table_config}. \n exc = {exc}')
 
 
     ## figure out the coordinate/dimension names programmatically TODO
 
     # Attempt to read lat coordinates
-    fre_logger.info(' attempting to read coordinate, lat')
+    fre_logger.info('attempting to read coordinate, lat')
     lat = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lat")
-    fre_logger.info(' attempting to read coordinate BNDS, lat_bnds')
+    fre_logger.info('attempting to read coordinate BNDS, lat_bnds')
     lat_bnds = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lat_bnds")
-    fre_logger.info(' attempting to read coordinate, lon')
+    fre_logger.info('attempting to read coordinate, lon')
     lon = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lon")
-    fre_logger.info(' attempting to read coordinate BNDS, lon_bnds')
+    fre_logger.info('attempting to read coordinate BNDS, lon_bnds')
     lon_bnds = from_dis_gimme_dis( from_dis  = ds,
                               gimme_dis = "lon_bnds")
 
     # read in time_coords + units
-    fre_logger.info(' attempting to read coordinate time, and units...')
+    fre_logger.info('attempting to read coordinate time, and units...')
     time_coords = from_dis_gimme_dis( from_dis = ds,
                                       gimme_dis = 'time' )
 
@@ -99,12 +99,12 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     fre_logger.info(f"                          time_coord_units = {time_coord_units}")
 
     # read in time_bnds , if present
-    fre_logger.info(' attempting to read coordinate BNDS, time_bnds')
+    fre_logger.info('attempting to read coordinate BNDS, time_bnds')
     time_bnds = from_dis_gimme_dis( from_dis = ds,
                                     gimme_dis = 'time_bnds' )
 
     # read the input variable data, i believe
-    fre_logger.info(f' attempting to read variable data, {target_var}')
+    fre_logger.info(f'attempting to read variable data, {target_var}')
     var = from_dis_gimme_dis( from_dis = ds,
                               gimme_dis = target_var )
 
@@ -149,13 +149,13 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
         # resolve location of statics file required for this processing.
         try:
-            fre_logger.info(f' netcdf_file is {netcdf_file}')
+            fre_logger.info(f'netcdf_file is {netcdf_file}')
             statics_file_path = find_statics_file(prev_path)
-            fre_logger.info(f' statics_file_path is {statics_file_path}')
+            fre_logger.info(f'statics_file_path is {statics_file_path}')
         except Exception as exc:
-            fre_logger.warning(' an ocean statics file is needed, but it could not be found.\n'
+            fre_logger.warning('an ocean statics file is needed, but it could not be found.\n'
                   '   moving on and doing my best, but i am probably going to break' )
-            raise Exception(' EXITING BC STATICS') from exc
+            raise Exception('EXITING BC STATICS') from exc
 
         fre_logger.info(" statics file found.")
         statics_file_name = Path(statics_file_path).name
@@ -164,14 +164,14 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         del statics_file_path
 
         statics_file_path = put_statics_file_here + '/' + statics_file_name
-        fre_logger.info(f' statics file path is now: {statics_file_path}')
+        fre_logger.info(f'statics file path is now: {statics_file_path}')
 
         # statics file read
         statics_ds = nc.Dataset(statics_file_path, 'r')
 
 
         # grab the lat/lon points, have shape (yh, xh)
-        fre_logger.info(' reading geolat and geolon coordinates of cell centers from statics file \n')
+        fre_logger.info('reading geolat and geolon coordinates of cell centers from statics file \n')
         statics_lat = from_dis_gimme_dis(statics_ds, 'geolat')#statics_ds['geolat'][:]#.copy()
         statics_lon = from_dis_gimme_dis(statics_ds, 'geolon')#statics_ds['geolon'][:]#.copy()
 
@@ -182,7 +182,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
         # spherical lat and lon coords
-        fre_logger.info(' creating lat and lon variables in temp file \n')
+        fre_logger.info('creating lat and lon variables in temp file \n')
         lat = ds.createVariable('lat', statics_lat.dtype, ('yh', 'xh') )
         lon = ds.createVariable('lon', statics_lon.dtype, ('yh', 'xh') )
         lat[:] = statics_lat[:]
@@ -195,7 +195,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
         # grab the corners of the cells, should have shape (yh+1, xh+1)
-        fre_logger.info(' reading geolat and geolon coordinates of cell corners from statics file \n')
+        fre_logger.info('reading geolat and geolon coordinates of cell corners from statics file \n')
         lat_c = from_dis_gimme_dis(statics_ds,'geolat_c')
         lon_c = from_dis_gimme_dis(statics_ds,'geolon_c')
 
@@ -206,13 +206,13 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
         # vertex
-        fre_logger.info(' creating vertex dimension\n')
+        fre_logger.info('creating vertex dimension\n')
         vertex = 4
         ds.createDimension('vertex', vertex)
 
 
         # lat and lon bnds
-        fre_logger.info(' creating lat and lon bnds from geolat and geolon of corners\n')
+        fre_logger.info('creating lat and lon bnds from geolat and geolon of corners\n')
         lat_bnds = ds.createVariable('lat_bnds', lat_c.dtype, ('yh', 'xh', 'vertex') )
         lat_bnds[:,:,0] = lat_c[1:,1:] # NE corner
         lat_bnds[:,:,1] = lat_c[1:,:-1] # NW corner
@@ -232,7 +232,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
 
         # grab the h-point lat and lon
-        fre_logger.info(' reading yh, xh\n')
+        fre_logger.info('reading yh, xh\n')
         yh = from_dis_gimme_dis(ds, 'yh')
         xh = from_dis_gimme_dis(ds, 'xh')
 
@@ -245,7 +245,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         xh_dim = len(xh)
 
         # read the q-point native-grid lat lon points
-        fre_logger.info(' reading yq, xq from statics file \n')
+        fre_logger.info('reading yq, xq from statics file \n')
         yq = from_dis_gimme_dis(statics_ds, 'yq')
         xq = from_dis_gimme_dis(statics_ds, 'xq')
 
@@ -258,7 +258,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
         assert xh_dim == (len(xq)-1)
 
         # create h-point bounds from the q-point lat lons
-        fre_logger.info(' creating yh_bnds, xh_bnds from yq, xq\n')
+        fre_logger.info('creating yh_bnds, xh_bnds from yq, xq\n')
 
         yh_bnds = ds.createVariable('yh_bnds', yq.dtype, ( 'yh', 'nv' ) )
         for i in range(0,yh_dim):
@@ -270,9 +270,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
             xh_bnds[i,0] = xq[i]
             xh_bnds[i,1] = xq[i+1]
             if i%200 == 0:
-                fre_logger.info(f' AFTER assignment: xh_bnds[{i}][0] = {xh_bnds[i][0]}')
-                fre_logger.info(f' AFTER assignment: xh_bnds[{i}][1] = {xh_bnds[i][1]}')
-                fre_logger.info(f'             type(xh_bnds[{i}][1]) = {type(xh_bnds[i][1])}')
+                fre_logger.info(f'AFTER assignment: xh_bnds[{i}][0] = {xh_bnds[i][0]}')
+                fre_logger.info(f'AFTER assignment: xh_bnds[{i}][1] = {xh_bnds[i][1]}')
+                fre_logger.info(f'            type(xh_bnds[{i}][1]) = {type(xh_bnds[i][1])}')
 
         fre_logger.info('\n')
         print_data_minmax(yh_bnds[:], "yh_bnds")
@@ -305,7 +305,7 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     json_grids_config, loaded_cmor_grids_cfg = None, None
     if process_tripolar_data:
         json_grids_config = str(Path(json_table_config).parent) + '/CMIP6_grids.json'
-        fre_logger.info(f' cmor is loading/opening {json_grids_config}')
+        fre_logger.info(f'cmor is loading/opening {json_grids_config}')
         loaded_cmor_grids_cfg =  cmor.load_table( json_grids_config )
         cmor.set_table(loaded_cmor_grids_cfg)
 
@@ -315,37 +315,37 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     # setup cmor latitude axis if relevant
     cmor_y = None
     if process_tripolar_data:
-        fre_logger.warning(' calling cmor.axis for a projected y coordinate!!')
+        fre_logger.warning('calling cmor.axis for a projected y coordinate!!')
         cmor_y = cmor.axis("y_deg", coord_vals = yh[:], cell_bounds = yh_bnds[:], units = "degrees")
     elif lat is None :
-        fre_logger.warning(' lat or lat_bnds is None, skipping assigning cmor_y')
+        fre_logger.warning('lat or lat_bnds is None, skipping assigning cmor_y')
     else:
-        fre_logger.info(' assigning cmor_y')
+        fre_logger.info('assigning cmor_y')
         if lat_bnds is None:
             cmor_y = cmor.axis("latitude", coord_vals = lat[:], units = "degrees_N")
         else:
             cmor_y = cmor.axis("latitude", coord_vals = lat[:], cell_bounds = lat_bnds, units = "degrees_N")
-        fre_logger.info('                          DONE assigning cmor_y')
+        fre_logger.info('                         DONE assigning cmor_y')
 
     # setup cmor longitude axis if relevant
     cmor_x = None
     if process_tripolar_data:
-        fre_logger.warning(' calling cmor.axis for a projected x coordinate!!')
+        fre_logger.warning('calling cmor.axis for a projected x coordinate!!')
         cmor_x = cmor.axis("x_deg", coord_vals = xh[:], cell_bounds = xh_bnds[:], units = "degrees")
     elif lon is None :
-        fre_logger.warning(' lon or lon_bnds is None, skipping assigning cmor_x')
+        fre_logger.warning('lon or lon_bnds is None, skipping assigning cmor_x')
     else:
-        fre_logger.info(' assigning cmor_x')
+        fre_logger.info('assigning cmor_x')
         cmor_x = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E")
         if lon_bnds is None:
             cmor_x = cmor.axis("longitude", coord_vals = lon[:], units = "degrees_E")
         else:
             cmor_x = cmor.axis("longitude", coord_vals = lon[:], cell_bounds = lon_bnds, units = "degrees_E")
-        fre_logger.info('                          DONE assigning cmor_x')
+        fre_logger.info('                         DONE assigning cmor_x')
 
     cmor_grid = None
     if process_tripolar_data:
-        fre_logger.warning(f' setting cmor.grid, process_tripolar_data = {process_tripolar_data}')
+        fre_logger.warning(f'setting cmor.grid, process_tripolar_data = {process_tripolar_data}')
         cmor_grid = cmor.grid( axis_ids = [cmor_y, cmor_x],
                                latitude = lat[:], longitude = lon[:],
                                latitude_vertices = lat_bnds[:],
@@ -356,20 +356,20 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
 
     # setup cmor time axis if relevant
     cmor_time = None
-    fre_logger.info(' assigning cmor_time')
+    fre_logger.info('assigning cmor_time')
     try: #if vert_dim != 'landuse':
         fre_logger.info(  " Executing cmor.axis('time', \n"
                f"    coord_vals = \n{time_coords}, \n"
                f"    cell_bounds = time_bnds, units = {time_coord_units})   ")
-        fre_logger.info(' assigning cmor_time using time_bnds...')
+        fre_logger.info('assigning cmor_time using time_bnds...')
         cmor_time = cmor.axis("time", coord_vals = time_coords,
                               cell_bounds = time_bnds, units = time_coord_units)
     except ValueError as exc: #else:
         fre_logger.info(" cmor_time = cmor.axis('time', \n"
                "                          coord_vals = time_coords, units = time_coord_units)")
-        fre_logger.info(' assigning cmor_time WITHOUT time_bnds...')
+        fre_logger.info('assigning cmor_time WITHOUT time_bnds...')
         cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units)
-    fre_logger.info('                          DONE assigning cmor_time')
+    fre_logger.info('                         DONE assigning cmor_time')
 
 
     # other vertical-axis-relevant initializations
@@ -381,10 +381,10 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     # set cmor vertical axis if relevant
     cmor_z = None
     if lev is not None:
-        fre_logger.info(' assigning cmor_z')
+        fre_logger.info('assigning cmor_z')
 
         if vert_dim.lower() in ["landuse", "plev39", "plev30", "plev19", "plev8", "height2m"]:
-            fre_logger.info(' non-hybrid sigma coordinate case')
+            fre_logger.info('non-hybrid sigma coordinate case')
             if vert_dim.lower() != "landuse":
                 cmor_vert_dim_name = vert_dim
                 cmor_z = cmor.axis( cmor_vert_dim_name,
@@ -395,15 +395,15 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
                 cmor_z = cmor.axis( cmor_vert_dim_name,
                                       coord_vals = np.array(
                                                             landuse_str_list,
-                                                            dtype=f'S{len(landuse_str_list[0])}' ),
+                                                            dtype=f'S{len(landuse_str_list[0])}'),
                                       units = lev_units )
 
 
         elif vert_dim in ["z_l"]:
             lev_bnds = create_lev_bnds( bound_these = lev,
                                          with_these = ds['z_i'] )
-            fre_logger.info(' created lev_bnds...')
-            fre_logger.info(f'  lev_bnds = \n{lev_bnds}')
+            fre_logger.info('created lev_bnds...')
+            fre_logger.info(f' lev_bnds = \n{lev_bnds}')
             cmor_z = cmor.axis( 'depth_coord',
                                   coord_vals = lev[:],
                                   units = lev_units,
@@ -449,53 +449,53 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
                                        zfactor_bounds = ds["b_bnds"][:],
                                        units          = ds["b"].units )
 
-            fre_logger.info(f' ierr_ap after calling cmor_zfactor: {ierr_ap}\n'
-                  f' ierr_b after calling cmor_zfactor: {ierr_b}'  )
+            fre_logger.info(f'ierr_ap after calling cmor_zfactor: {ierr_ap}\n'
+                  f'ierr_b after calling cmor_zfactor: {ierr_b}'  )
             axis_ids = []
             if cmor_time is not None:
-                fre_logger.info(' appending cmor_time to axis_ids list...')
+                fre_logger.info('appending cmor_time to axis_ids list...')
                 axis_ids.append(cmor_time)
-                fre_logger.info(f'    axis_ids now = {axis_ids}')
+                fre_logger.info(f'   axis_ids now = {axis_ids}')
             # might there need to be a conditional check for tripolar ocean data here as well? TODO
             if cmor_y is not None:
-                fre_logger.info(' appending cmor_y to axis_ids list...')
+                fre_logger.info('appending cmor_y to axis_ids list...')
                 axis_ids.append(cmor_y)
-                fre_logger.info(f'    axis_ids now = {axis_ids}')
+                fre_logger.info(f'   axis_ids now = {axis_ids}')
             if cmor_x is not None:
-                fre_logger.info(' appending cmor_x to axis_ids list...')
+                fre_logger.info('appending cmor_x to axis_ids list...')
                 axis_ids.append(cmor_x)
-                fre_logger.info(f'    axis_ids now = {axis_ids}')
+                fre_logger.info(f'   axis_ids now = {axis_ids}')
 
             ips = cmor.zfactor( zaxis_id     = cmor_z,
                                 zfactor_name = "ps",
                                 axis_ids     = axis_ids, #[cmor_time, cmor_y, cmor_x],
                                 units        = "Pa" )
             save_ps = True
-        fre_logger.info('    DONE assigning cmor_z')
+        fre_logger.info('   DONE assigning cmor_z')
 
 
     axes = []
     if cmor_time is not None:
-        fre_logger.info(' appending cmor_time to axes list...')
+        fre_logger.info('appending cmor_time to axes list...')
         axes.append(cmor_time)
-        fre_logger.info(f'    axes now = {axes}')
+        fre_logger.info(f'   axes now = {axes}')
 
     if cmor_z is not None:
-        fre_logger.info(' appending cmor_z to axes list...')
+        fre_logger.info('appending cmor_z to axes list...')
         axes.append(cmor_z)
-        fre_logger.info(f'    axes now = {axes}')
+        fre_logger.info(f'   axes now = {axes}')
 
     if process_tripolar_data:
         axes.append(cmor_grid)
     else:
         if cmor_y is not None:
-            fre_logger.info(' appending cmor_y to axes list...')
+            fre_logger.info('appending cmor_y to axes list...')
             axes.append(cmor_y)
-            fre_logger.info(f'    axes now = {axes}')
+            fre_logger.info(f'   axes now = {axes}')
         if cmor_x is not None:
-            fre_logger.info(' appending cmor_x to axes list...')
+            fre_logger.info('appending cmor_x to axes list...')
             axes.append(cmor_x)
-            fre_logger.info(f'    axes now = {axes}')
+            fre_logger.info(f'   axes now = {axes}')
 
 
     # read positive/units attribute and create cmor_var
@@ -511,9 +511,9 @@ def rewrite_netcdf_file_var ( proj_table_vars = None,
     cmor.write(cmor_var, var)
     if save_ps:
         if any( [ ips is None, ps is None ] ):
-            fre_logger.warning( ' ps or ips is None!, but save_ps is True!\n'
-                  f'    ps = {ps}, ips = {ips}\n'
-                   '    skipping ps writing!'    )
+            fre_logger.warning( 'ps or ips is None!, but save_ps is True!\n'
+                  f'   ps = {ps}, ips = {ips}\n'
+                   '   skipping ps writing!'   )
         else:
             cmor.write(ips, ps, store_with = cmor_var)
             cmor.close(ips, file_name = True, preserve = False)
@@ -614,8 +614,8 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
                                                        json_table_config    , nc_fls[i]  )
         except Exception as exc:
             raise Exception('(cmorize_target_var_files) problem with rewrite_netcdf_file_var. exc=\n'
-                            f'                           {exc}\n'
-                            '                           exiting and executing finally block.')
+                            f'                          {exc}\n'
+                            '                          exiting and executing finally block.')
         finally: # should always execute, errors or not!
             fre_logger.warning(f'changing directory to: \n      {gotta_go_back_here}')
             os.chdir( gotta_go_back_here )
@@ -634,7 +634,7 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
             fre_logger.info(f'attempting to create filedir={filedir}')
             os.makedirs(filedir)
         except FileExistsError:
-            fre_logger.warning(f' directory {filedir} already exists!')
+            fre_logger.warning(f'directory {filedir} already exists!')
 
         # hmm.... this is making issues for pytest
         mv_cmd = f"mv {tmp_dir}/{local_file_name} {filedir}"
@@ -646,7 +646,7 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
         filename_no_nc = filename[:filename.rfind(".nc")]
         chunk_str = filename_no_nc[-6:]
         if not chunk_str.isdigit():
-            fre_logger.warning(f' chunk_str is not a digit: '
+            fre_logger.warning(f'chunk_str is not a digit: '
                   f'chunk_str = {chunk_str}')
             filename_corr = "{filename[:filename.rfind('.nc')]}_{iso_datetime}.nc"
             mv_cmd = f"mv {filename} {filename_corr}"
diff --git a/fre/fre.py b/fre/fre.py
index b935f82c..880543f3 100644
--- a/fre/fre.py
+++ b/fre/fre.py
@@ -14,6 +14,8 @@
 
 import logging
 fre_logger = logging.getLogger(__name__)
+FORMAT = "%(levelname)s:%(filename)s:%(funcName)s %(message)s"
+MODE = 'x'
 
 # versioning, turn xxxx.y into xxxx.0y
 version_unexpanded = importlib.metadata.version('fre-cli')
@@ -52,12 +54,18 @@
 @click.option('-v', '--verbose', is_flag=True,
               default=False, help="set logging verbosity higher",
               required=False)
-def fre(verbose):
+@click.option('-l', '--log_file', default=None, required=False, type=str,
+              help='path to log file for all fre calls. leave as None to print to screen')
+def fre(verbose, log_file):
     ''' entry point function to subgroup functions '''
+    log_level=None
+    file_mode = None if log_file is None else MODE
     if verbose:
-        logging.basicConfig(level=logging.INFO)
+        log_level=logging.INFO
     else:
-        logging.basicConfig(level=logging.WARN)
+        log_level=logging.WARN
+    logging.basicConfig(level=log_level, format=FORMAT,
+                        filename=log_file, encoding='utf-8')
 
 if __name__ == '__main__':
     fre()
diff --git a/fre/pytest.ini b/fre/pytest.ini
index 80ef0c32..2067ada2 100644
--- a/fre/pytest.ini
+++ b/fre/pytest.ini
@@ -9,3 +9,8 @@ testpaths =
     fre/make/tests
     fre/pp/tests
     fre/yamltools/tests
+
+# https://docs.pytest.org/en/stable/how-to/logging.html
+log_cli = True
+# https://docs.python.org/3/library/logging.html#levels
+log_cli_level = 20
\ No newline at end of file

From f837840516566510d272dbf5f24dc94b3dcb74b6 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 17 Jan 2025 17:03:25 -0500
Subject: [PATCH 06/23] see if this keeps the pipeline from being grumpy...
 maybe TOO much logging stdout to screen!

---
 fre/pytest.ini | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fre/pytest.ini b/fre/pytest.ini
index 2067ada2..803efc5b 100644
--- a/fre/pytest.ini
+++ b/fre/pytest.ini
@@ -11,6 +11,8 @@ testpaths =
     fre/yamltools/tests
 
 # https://docs.pytest.org/en/stable/how-to/logging.html
-log_cli = True
+# interestingly enough, if this is true, the testing pipeline fails. why?
+#log_cli = True
+
 # https://docs.python.org/3/library/logging.html#levels
-log_cli_level = 20
\ No newline at end of file
+#log_cli_level = 20
\ No newline at end of file

From 03acb95c56779ea5777530e0d8ce08f04f40706d Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Wed, 22 Jan 2025 12:37:50 -0500
Subject: [PATCH 07/23] update git ignore

---
 .gitignore | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5212a9a5..01e88e96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,3 @@
-# fremake generated files that should be ignored
-combined-null_model.yaml
-combined-am5.yaml
-tmp
-makefile_out
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -60,12 +55,7 @@ coverage.xml
 .pytest_cache/
 cover/
 
-# files pulled/created via testing
-fre/tests/test_files/cmip6-cmor-tables/
-fre/tests/test_files/outdirfre/
-fre/pp/tests/configure_yaml_out/
-fre/yamltools/tests/combine_yamls_out/
-fre/app/generate_time_averages/tests/time_avg_test_files/
+
 
 # Translations
 *.mo
@@ -170,10 +160,28 @@ cython_debug/
 
 # other
 *~
+*\#
 
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+
+
+
+
+# files pulled/created via testing
+fre/tests/test_files/outdirfre/
+fre/pp/tests/configure_yaml_out/
+fre/yamltools/tests/combine_yamls_out/
+fre/app/generate_time_averages/tests/time_avg_test_files/
+
+# fremake generated files that should be ignored
+combined-null_model.yaml
+combined-am5.yaml
+tmp
+makefile_out
+fre/tests/test_files/ocean_sos_var_file/*.nc
+fre/tests/test_files/outdir/
\ No newline at end of file

From 4ce479ab99b06ef33166261f3bb7cd642268e4e5 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 11:42:11 -0500
Subject: [PATCH 08/23] prelim cmor yaml and cmor_yamler, very rough, but lets
 get back into it- shall we?

---
 fre/cmor/cmor.yaml                       | 38 ++++++++++++++++++++++++
 fre/cmor/cmor_yamler.py                  | 15 ++++++++++
 fre/gfdl_msd_schemas                     |  2 +-
 fre/yamltools/tests/AM5_example/am5.yaml |  2 ++
 4 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 fre/cmor/cmor.yaml
 create mode 100644 fre/cmor/cmor_yamler.py

diff --git a/fre/cmor/cmor.yaml b/fre/cmor/cmor.yaml
new file mode 100644
index 00000000..977cab82
--- /dev/null
+++ b/fre/cmor/cmor.yaml
@@ -0,0 +1,38 @@
+
+
+
+# where we are
+cmor:
+  indir: ''
+  json_var_list: ''
+  json_table_config: ''
+  json_exp_config: ''
+  outdir: ''
+  opt_var_name: '' #[OPTIONAL SPECIAL-CASE ORIENTED USAGE]
+
+
+
+
+
+
+## from the test cases. 
+## Chris's desired end-state (best guess)
+#cmor:
+#  global:
+#    parent_experiment_name:
+#    xxxx
+#  Amon-tas-gr1:
+#    table: Amon
+#    variable: tas
+#    grid: gr1
+#    gfdl_comp: atmos
+#    gfdl_variable: t_ref
+#        '/archive/Eric.Stofferahn/ ' + \ #user
+#         'CMIP7/ESM4/DEV/' + \ #FRE stem
+#          'ESM4.5v01_om5b04_piC/' + \ #exp name
+#          'gfdl.ncrc5-intel23-prod-openmp/' + \ #platform
+#        'pp/' + \ #postprocess
+#        'land/' + \ # GFDL-component name
+#        'ts/' + \ # time-series
+#        'monthly/' + \ #frequency 
+#        '5yr/' # span of itme?
diff --git a/fre/cmor/cmor_yamler.py b/fre/cmor/cmor_yamler.py
new file mode 100644
index 00000000..292ea465
--- /dev/null
+++ b/fre/cmor/cmor_yamler.py
@@ -0,0 +1,15 @@
+import yaml
+
+from .cmor_mixer import cmor_run_subtool
+
+def cmor_yamler_subtool(yamlfile):
+
+    yaml_data = yaml.load(yamlfile, 'r') # check that
+
+    arg1 = yaml_data.access('KEY1' )
+    
+    cmor_run_subtool(
+        #        opt_var_name = arg1
+    )
+
+    return
diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas
index 32b9d7ca..04c8150b 160000
--- a/fre/gfdl_msd_schemas
+++ b/fre/gfdl_msd_schemas
@@ -1 +1 @@
-Subproject commit 32b9d7ca00aa314b781341dda4d241d48e588d18
+Subproject commit 04c8150bc362304d82e60e765405135460b69f06
diff --git a/fre/yamltools/tests/AM5_example/am5.yaml b/fre/yamltools/tests/AM5_example/am5.yaml
index 1cb07353..97b61ac7 100644
--- a/fre/yamltools/tests/AM5_example/am5.yaml
+++ b/fre/yamltools/tests/AM5_example/am5.yaml
@@ -74,3 +74,5 @@ experiments:
     analysis:
       - "analysis_yamls/clouds.yaml"
       - "analysis_yamls/land.yaml"
+    cmor: # cmor-tool development, not part of official am5.yaml (yet)
+      - "cmor_yamls/cmor.am5.yaml"

From 086b92597b48b36c1a97f2954306529158ce9745 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 12:12:43 -0500
Subject: [PATCH 09/23] unify logging module usage, at least partially.
 subtools are still allowed to set FORMAT and logging verbosity at this time

---
 fre/make/create_checkout_script.py |  13 ++--
 fre/make/create_compile_script.py  |   8 ++-
 fre/make/run_fremake_script.py     |  15 ++--
 fre/pp/configure_script_xml.py     | 109 ++++++++++++++---------------
 4 files changed, 73 insertions(+), 72 deletions(-)

diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py
index ed2559ef..2635270b 100644
--- a/fre/make/create_checkout_script.py
+++ b/fre/make/create_checkout_script.py
@@ -6,6 +6,9 @@
 import os
 import subprocess
 import logging
+fre_logger = logging.getLogger(__name__)
+
+
 import sys
 
 import fre.yamltools.combine_yamls as cy
@@ -25,9 +28,9 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec
         pc = " &"
 
     if verbose:
-        logging.basicConfig(level=logging.INFO)
+        fre_logger.setLevel(level = logging.INFO)
     else:
-        logging.basicConfig(level=logging.ERROR)
+        fre_logger.setLevel(level = logging.ERROR)
 
     src_dir="src"
     checkout_script_name = "checkout.sh"
@@ -78,7 +81,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec
                 fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc)
                 # Make checkout script executable
                 os.chmod(src_dir+"/checkout.sh", 0o744)
-                print("\nCheckout script created in "+ src_dir + "/checkout.sh \n")
+                logging.info("\nCheckout script created in "+ src_dir + "/checkout.sh \n")
 
                 # Run the checkout script
                 if run:
@@ -86,7 +89,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec
                 else:
                     return
             else:
-                print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n")
+                logging.info("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n")
                 if run:
                     try:
                         subprocess.run(args=[src_dir+"/checkout.sh"], check=True)
@@ -104,7 +107,7 @@ def checkout_create(yamlfile, platform, target, no_parallel_checkout, jobs, exec
             fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
             fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
             fre_checkout.finish(model_yaml.compile.getCompileYaml(),pc)
-            print("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n")
+            logging.info("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n")
 
 if __name__ == "__main__":
     checkout_create()
diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py
index 216ee12a..f5ad35a5 100644
--- a/fre/make/create_compile_script.py
+++ b/fre/make/create_compile_script.py
@@ -4,6 +4,8 @@
 import os
 import sys
 import logging
+fre_logger = logging.getLogger(__name__)
+
 from pathlib import Path
 from multiprocessing.dummy import Pool
 
@@ -18,9 +20,9 @@ def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose)
     jobs = str(jobs)
 
     if verbose:
-      logging.basicCOnfig(level=logging.INFO)
+        fre_logger.setLevel(level = logging.INFO)
     else:
-      logging.basicConfig(level=logging.ERROR)
+        fre_logger.setLevel(level = logging.ERROR)
 
     srcDir="src"
     checkoutScriptName = "checkout.sh"
@@ -79,7 +81,7 @@ def compile_create(yamlfile, platform, target, jobs, parallel, execute, verbose)
                    fremakeBuild.writeBuildComponents(c)
               fremakeBuild.writeScript()
               fremakeBuildList.append(fremakeBuild)
-              print("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
+              logging.info("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
 
     if execute:
         if baremetalRun:
diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py
index 6d2e0383..ae5bff19 100644
--- a/fre/make/run_fremake_script.py
+++ b/fre/make/run_fremake_script.py
@@ -7,6 +7,9 @@
 
 import os
 import logging
+fre_logger = logging.getLogger(__name__)
+
+
 from multiprocessing.dummy import Pool
 from pathlib import Path
 
@@ -30,9 +33,9 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
         pc = " &"
 
     if verbose:
-        logging.basicConfig(level=logging.INFO)
+        fre_logger.setLevel(level = logging.INFO)
     else:
-        logging.basicConfig(level=logging.ERROR)
+        fre_logger.setLevel(level = logging.ERROR)
 
     #### Main
     srcDir="src"
@@ -84,7 +87,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                 freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
                 freCheckout.finish(modelYaml.compile.getCompileYaml(),pc)
                 os.chmod(srcDir+"/checkout.sh", 0o744)
-                print("\nCheckout script created at "+ srcDir + "/checkout.sh \n")
+                logging.info("\nCheckout script created at "+ srcDir + "/checkout.sh \n")
                 ## TODO: Options for running on login cluster?
                 freCheckout.run()
 
@@ -121,7 +124,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                 # Loop through components, send component name/requires/overrides for Makefile
                 for c in fremakeYaml['src']:
                     freMakefile.addComponent(c['component'], c['requires'], c['makeOverrides'])
-                print("\nMakefile created at " + bldDir + "/Makefile" + "\n")
+                logging.info("\nMakefile created at " + bldDir + "/Makefile" + "\n")
                 freMakefile.writeMakefile()
 
                 ## Create a list of compile scripts to run in parallel
@@ -142,7 +145,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                 if execute:
                     fremakeBuild.run()
                 else:
-                    print("Compile script created at "+ bldDir+"/compile.sh\n\n")
+                    logging.info("Compile script created at "+ bldDir+"/compile.sh\n\n")
             else:
                 ###################### container stuff below #######################################
                 ## Run the checkout script
@@ -188,7 +191,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
 
                 # Create build script for container
                 dockerBuild.createBuildScript(platform["containerBuild"], platform["containerRun"])
-                print("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
+                logging.info("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
 
                 # Execute if flag is given
                 if execute:
diff --git a/fre/pp/configure_script_xml.py b/fre/pp/configure_script_xml.py
index 1cf84340..a1977c09 100644
--- a/fre/pp/configure_script_xml.py
+++ b/fre/pp/configure_script_xml.py
@@ -12,6 +12,7 @@
 import os
 import subprocess
 import logging
+fre_logger = logging.getLogger(__name__)
 
 # third party
 import metomi.rose.config
@@ -68,13 +69,13 @@ def chunk_from_legacy(legacy_chunk):
     regex = re.compile( r'(\d+)(\w+)' )
     match = regex.match( legacy_chunk )
     if not match:
-        logging.error( "Could not convert Bronx chunk to ISO8601 duration: %s",
+        fre_logger.error( "Could not convert Bronx chunk to ISO8601 duration: %s",
                        legacy_chunk )
         raise ValueError
 
     time_unit = match.group(2)
     if time_unit not in ['yr','mo']:
-        logging.error("Unknown time units %s", match.group(2) )
+        fre_logger.error("Unknown time units %s", match.group(2) )
         raise ValueError
 
     time_quant = match.group(1)
@@ -98,17 +99,17 @@ def frelist_xpath(xml, platform, target, experiment,
     """
 
     cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} --evaluate '{xpath}'"
-    logging.info("running cmd:\n %s",cmd)   #logging.info(f"running cmd:\n {cmd}")
-    logging.info(">> %s",xpath)
+    fre_logger.info("running cmd:\n %s",cmd)   #fre_logger.info(f"running cmd:\n {cmd}")
+    fre_logger.info(">> %s",xpath)
     process = subprocess.run(cmd,
                              shell=True,
                              check=False, #if True, retrieving std err difficult...
                              capture_output=True,
                              universal_newlines=True)
 
-    logging.info("stdout: \n%s",process.stdout.strip())
-    logging.info("stderr: \n%s",process.stderr.strip())
-    logging.info("returncode: \n%s",process.returncode)
+    fre_logger.info("stdout: \n%s",process.stdout.strip())
+    fre_logger.info("stderr: \n%s",process.stderr.strip())
+    fre_logger.info("returncode: \n%s",process.returncode)
 
 
     result = process.stdout.strip()
@@ -192,11 +193,11 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
     # the XML's default paths will be inserted into rose-suite.conf.
     ##########################################################################
     if historydir is None:
-        logging.info("Running frelist for historydir assignment, this may fail")
-        logging.info("If so, try the frelist call manually and use the historydir argument")
+        fre_logger.info("Running frelist for historydir assignment, this may fail")
+        fre_logger.info("If so, try the frelist call manually and use the historydir argument")
 
         fetch_history_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d archive"
-        logging.info(">> %s", fetch_history_cmd)
+        fre_logger.info(">> %s", fetch_history_cmd)
 
         fetch_history_process = subprocess.run(fetch_history_cmd,
                                                shell=True,
@@ -204,7 +205,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
                                                capture_output=True,
                                                universal_newlines=True)
         historydir = fetch_history_process.stdout.strip() + '/history'
-        logging.info(historydir)
+        fre_logger.info(historydir)
 
 
 
@@ -212,10 +213,10 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
     historydir_refined = historydir + '_refineDiag'
 
     if ppdir is None:
-        logging.info("Running frelist for ppdir assignment...")
-        logging.info("If this fails, try the frelist call manually and use the ppdir argument")
+        fre_logger.info("Running frelist for ppdir assignment...")
+        fre_logger.info("If this fails, try the frelist call manually and use the ppdir argument")
         fetch_pp_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d postProcess"
-        logging.info(">> %s", fetch_pp_cmd)
+        fre_logger.info(">> %s", fetch_pp_cmd)
 
         fetch_pp_process = subprocess.run(fetch_pp_cmd,
                                           shell=True,
@@ -223,15 +224,15 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
                                           capture_output=True,
                                           universal_newlines=True)
         ppdir = fetch_pp_process.stdout.strip()
-        logging.info(ppdir)
+        fre_logger.info(ppdir)
 
 
     # Q: shouldn't there be a CLI analysis dir arg while we're here?
     # basically, this is borderline on the same level as the ppDir and historydir fields.
     #if do_analysis:
-    logging.info("Running frelist for analysis_dir assignment...")
+    fre_logger.info("Running frelist for analysis_dir assignment...")
     fetch_analysis_dir_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d analysis"
-    logging.info(">> %s", fetch_analysis_dir_cmd)
+    fre_logger.info(">> %s", fetch_analysis_dir_cmd)
 
     fetch_analysis_dir_process = subprocess.run(fetch_analysis_dir_cmd,
                                                 shell=True,
@@ -239,9 +240,9 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
                                             capture_output=True,
                                                 universal_newlines=True)
     analysis_dir = fetch_analysis_dir_process.stdout.strip()
-    logging.info(analysis_dir)
+    fre_logger.info(analysis_dir)
     #else:
-    #    logging.info('not doing analysis.')
+    #    fre_logger.info('not doing analysis.')
 
 
     grid_spec = frelist_xpath(xml, platform, target, experiment,
@@ -329,20 +330,20 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
         rose_suite.set(keys=['template variables', '#REFINEDIAG_SCRIPTS'],
                        value=f"'{' '.join(list_refinediags)}'")
 
-        logging.info( "refineDiag scripts: %s", ' '.join(list_refinediags) )
-        logging.info( "NOTE: Now turned off by default; please re-enable in config file if needed" )
+        fre_logger.info( "refineDiag scripts: %s", ' '.join(list_refinediags) )
+        fre_logger.info( "NOTE: Now turned off by default; please re-enable in config file if needed" )
     else:
         rose_suite.set(keys=['template variables', 'DO_REFINEDIAG'], value='False')
-        logging.info("No refineDiag scripts written. ")
+        fre_logger.info("No refineDiag scripts written. ")
 
     if str_preanalysis is not None:
         rose_suite.set(keys=['template variables', 'DO_PREANALYSIS'], value='True')
         rose_suite.set(keys=['template variables', 'PREANALYSIS_SCRIPT'],
                        value=f"'{str_preanalysis}'" )
-        logging.info( "Preanalysis script: %s", str_preanalysis )
+        fre_logger.info( "Preanalysis script: %s", str_preanalysis )
     else:
         rose_suite.set(keys=['template variables', 'DO_PREANALYSIS'], value='False')
-        logging.info( "No preAnalysis scripts written." )
+        fre_logger.info( "No preAnalysis scripts written." )
 
     # Grab all of the necessary PP component items/elements from the XML
     comps = frelist_xpath(xml, platform, target, experiment,
@@ -372,7 +373,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
     elif segment_units == 'months':
         segment = f'P{segment_time}M'
     else:
-        logging.error("Unknown segment units: %s", segment_units)
+        fre_logger.error("Unknown segment units: %s", segment_units)
         raise ValueError
 
     # P12M is identical to P1Y but the latter looks nicer
@@ -394,12 +395,12 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
             current_date = metomi.isodatetime.data.TimePoint(
                 year=match.group(1), month_of_year=match.group(2), day_of_month=match.group(3) )
         except:
-            logging.warning("Could not parse date from namelist current_date")
+            fre_logger.warning("Could not parse date from namelist current_date")
             current_date = None
     else:
         current_date = None
-        logging.warning("Could not find current_date in namelists")
-    logging.info("current_date (from namelists): %s", current_date)
+        fre_logger.warning("Could not find current_date in namelists")
+    fre_logger.info("current_date (from namelists): %s", current_date)
 
     # Take a good guess for the PP_START and PP_STOP
     # PP_START could be the coupler_nml/current_date
@@ -423,7 +424,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
     for comp in comps:
         comp_count += 1
         pp_comp_xpath_header = f'postProcess/component[@type="{comp}"]'
-        logging.info( "Component loop: %s out of %s", comp_count, len(comps))
+        fre_logger.info( "Component loop: %s out of %s", comp_count, len(comps))
 
         # get the comp attributes
         comp_source = frelist_xpath(xml, platform, target, experiment,
@@ -480,7 +481,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
 
         sources = set()
         if comp_source.endswith('_refined'):
-            logging.info(
+            fre_logger.info(
                 "NOTE: Skipping history file %s, refineDiag is turned off by default.", comp_source)
         else:
             sources.add(comp_source)
@@ -507,7 +508,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
                                    #verbose, quiet, dual,
                                    f'{pp_comp_xpath_header}/timeSeries[{i}]/@source' )
             if source.endswith('_refined'):
-                logging.info(
+                fre_logger.info(
                     "NOTE: Skipping history file %s, refineDiag is turned off by default.", source)
             else:
                 sources.add(source)
@@ -543,9 +544,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
             rose_regrid_xy.set(keys=[comp, 'outputGridLat'], value=output_grid_lat)
 
     # Process all of the found PP chunks into the rose-suite configuration
-    if verbose:
-        print("")
-    logging.info("Setting PP chunks...")
+    fre_logger.info("Setting PP chunks...")
 
     sorted_chunks = list(chunks)
     sorted_chunks.sort(key=duration_to_seconds, reverse=False)
@@ -553,7 +552,7 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
     if len(chunks) == 0:
         raise ValueError('no chunks found! exit.')
 
-    logging.info("  Chunks found: %s", ', '.join(sorted_chunks))
+    fre_logger.info("  Chunks found: %s", ', '.join(sorted_chunks))
     if len(chunks) == 1:
         rose_suite.set(['template variables', 'PP_CHUNK_A'],
                        f"'{sorted_chunks[0]}'")
@@ -562,27 +561,25 @@ def main(xml, platform, target, experiment, do_analysis, historydir, refinedir,
                        f"'{sorted_chunks[0]}'")
         rose_suite.set(['template variables', 'PP_CHUNK_B'],
                        f"'{sorted_chunks[1]}'")
-    logging.info("  Chunks used: %s", ', '.join(sorted_chunks[0:2]) )
+    fre_logger.info("  Chunks used: %s", ', '.join(sorted_chunks[0:2]) )
 
     # Write out the final configurations.
-    if verbose:
-        print("")
-    logging.info("Writing output files...")
+    fre_logger.info("Writing output files...")
 
     dumper = metomi.rose.config.ConfigDumper()
     rose_outfiles=[ ( rose_suite,     'rose-suite.conf'                       ),
                     ( rose_remap,     'app/remap-pp-components/rose-app.conf' ),
                     ( rose_regrid_xy, 'app/regrid-xy/rose-app.conf'           ) ]
     for outfile in rose_outfiles:
-        logging.info("  %s", outfile[1])
+        fre_logger.info("  %s", outfile[1])
         dumper(outfile[0], outfile[1])
 
     #outfile = "app/remap-pp-components/rose-app.conf"
-    #logging.info("  %s", outfile)
+    #fre_logger.info("  %s", outfile)
     #dumper(rose_remap, outfile)
     #
     #outfile = "app/regrid-xy/rose-app.conf"
-    #logging.info("  %s", outfile)
+    #fre_logger.info("  %s", outfile)
     #dumper(rose_regrid_xy, outfile)
 
 def _convert(xml, platform, target, experiment, do_analysis=False, historydir=None,
@@ -594,11 +591,11 @@ def _convert(xml, platform, target, experiment, do_analysis=False, historydir=No
 
     # Logging settings. The default is throwing only warning messages
     if verbose:
-        logging.basicConfig(level=logging.INFO, format=LOGGING_FORMAT)
+        fre_logger.setLevel( level = logging.INFO )
     elif quiet:
-        logging.basicConfig(level=logging.ERROR, format=LOGGING_FORMAT)
+        fre_logger.setLevel( level = logging.ERROR )
     else:
-        logging.basicConfig(level=logging.WARNING, format=LOGGING_FORMAT)
+        fre_logger.setLevel( level = logging.WARNING )
 
     # Set the name of the directory
     name = f"{experiment}__{platform}__{target}"
@@ -630,10 +627,10 @@ def _convert(xml, platform, target, experiment, do_analysis=False, historydir=No
     # rose-suite configuration
     if any( [ pp_stop is None, pp_start is None ] ):
         if pp_start is None:
-            logging.warning("PP start year was not specified.")
+            fre_logger.warning("PP start year was not specified.")
         if pp_stop is None:
-            logging.warning("PP stop year was not specified.")
-        logging.warning("After the converter has run, please edit the "                \
+            fre_logger.warning("PP stop year was not specified.")
+        fre_logger.warning("After the converter has run, please edit the "                \
                         "default '0000' values within your rose-suite.conf.")
 
     # These series of conditionals takes into account input from the user
@@ -649,37 +646,33 @@ def format_req_pp_year(pp_year):
         pp_stop = format_req_pp_year(pp_stop)
 
         if int(pp_start) >= int(pp_stop):
-            logging.warning("Your PP_START date is equal to or later than "            \
+            fre_logger.warning("Your PP_START date is equal to or later than "            \
                             "your PP_STOP date. Please revise these values in "        \
                             "your configuration after the converter has run.")
         if any( [ len(pp_start) >  4, len(pp_stop) >  4,
                   int(pp_start) <= 0, int(pp_stop) <= 0 ] ):
-            logging.warning("At least one of your PP_start or PP_stop years "          \
+            fre_logger.warning("At least one of your PP_start or PP_stop years "          \
                             "does not make sense. Please revise this value in "        \
                             "your configuration after the converter has run.")
 
     main( xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir,
           do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual )
 
-    if verbose:
-        print("")
-    logging.info("XML conversion complete!")
+    fre_logger.info("XML conversion complete!")
 
     # Run the Cylc validator tool on the current directory if conditions are met.
     # Note: the user must be running the converter in the parent Cylc Workflow
     # Directory if the validator is run.
     if cylc_loaded:
-        if verbose:
-            print("")
-        logging.info("Running the Cylc validator tool...")
+        fre_logger.info("Running the Cylc validator tool...")
         try:
             subprocess.run("cylc validate .", shell=True, check=True)
         except subprocess.CalledProcessError:
-            logging.error("Errant values in rose-suite.conf or other Cylc errors. "    \
+            fre_logger.error("Errant values in rose-suite.conf or other Cylc errors. "    \
                             "Please check your configuration and run the validator "     \
                             "again separately.")
         finally:
-            logging.info("Validation step complete!")
+            fre_logger.info("Validation step complete!")
 
 ##############################################
 

From ce9e8a6a672da1c0d423312e1b6a32f3880e2c3b Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 12:31:11 -0500
Subject: [PATCH 10/23] touch up the fre cmor yaml routine bits- bones and some
 other things. playing with some imports

---
 fre/cmor/__init__.py    |   3 --
 fre/cmor/cmor_yamler.py |   5 +-
 fre/cmor/frecmor.py     | 100 +++++++++++++++++++++++++---------------
 3 files changed, 67 insertions(+), 41 deletions(-)

diff --git a/fre/cmor/__init__.py b/fre/cmor/__init__.py
index 4bb50201..e69de29b 100644
--- a/fre/cmor/__init__.py
+++ b/fre/cmor/__init__.py
@@ -1,3 +0,0 @@
-''' for fre.cmor imports '''
-from .cmor_mixer import cmor_run_subtool
-from .cmor_finder import cmor_find_subtool
diff --git a/fre/cmor/cmor_yamler.py b/fre/cmor/cmor_yamler.py
index 292ea465..fc1b0f90 100644
--- a/fre/cmor/cmor_yamler.py
+++ b/fre/cmor/cmor_yamler.py
@@ -2,7 +2,10 @@
 
 from .cmor_mixer import cmor_run_subtool
 
-def cmor_yamler_subtool(yamlfile):
+def cmor_yaml_subtool(yamlfile):
+    '''
+    the thing that carries out the cmorization yamlerization
+    '''
 
     yaml_data = yaml.load(yamlfile, 'r') # check that
 
diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py
index 40505dcf..1e96e76d 100644
--- a/fre/cmor/frecmor.py
+++ b/fre/cmor/frecmor.py
@@ -4,6 +4,7 @@
 
 from .cmor_finder import cmor_find_subtool
 from .cmor_mixer import cmor_run_subtool
+from .cmor_yamler import cmor_yaml_subtool
 
 OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \
                   "matching that variable name. I.e., this string help target local_vars, not " + \
@@ -17,6 +18,68 @@
 def cmor_cli():
     ''' entry point to fre cmor click commands '''
 
+
+@cmor_cli.command()
+@click.option("-y", "--yamlfile",
+              type=str, default = None,
+              help='YAML file to be used for parsing',
+              required=True)
+def yaml(yamlfile):
+    '''
+    fre cmor yamler gonna yaml. 
+    where your cmorization yaml gets yamled by the cmor yamler. 
+    i yaml what i yaml and that's all that i aml
+    i could do this all day. 
+    shoutout to my federal home PEOPLE.
+    '''
+
+    # if opt_var_name specified, forget the list.
+    if yamlfile is None:
+        raise ValueError('I need a yamlfile!!!')
+
+    cmor_yamler_subtool(
+        yamlfile = yamlfile
+    )
+
+
+@cmor_cli.command()
+@click.option("-l", "--varlist",
+              type=str,
+              help=VARLIST_HELP,
+              required=False)
+@click.option("-r", "--table_config_dir",
+              type=str,
+              help="directory holding MIP tables to search for variables in var list",
+              required=True)
+@click.option('-v', "--opt_var_name",
+              type = str,
+              help=OPT_VAR_NAME_HELP,
+              required=False)
+def find(varlist, table_config_dir, opt_var_name):
+    '''
+    loop over json table files in config_dir and show which tables contain variables in var list/
+    the tool will also print what that table entry is expecting of that variable as well. if given
+    an opt_var_name in addition to varlist, only that variable name will be printed out.
+    accepts 3 arguments, two of the three required.
+    '''
+
+    # if opt_var_name specified, forget the list.
+    if opt_var_name is not None:
+        varlist=None
+
+    # custom arg requirement of "one of the two or both" in click should be implemented with
+    # logic before calling context.invoke( <thingy>, *args )
+    if opt_var_name is None and varlist is None:
+        raise ValueError('opt_var_name and varlist cannot both be None')
+
+    cmor_find_subtool(
+        json_var_list = varlist,
+        json_table_config_dir = table_config_dir,
+        opt_var_name = opt_var_name
+    )
+
+
+
 @cmor_cli.command()
 @click.option("-d", "--indir",
               type=str,
@@ -62,42 +125,5 @@ def run(indir, varlist, table_config, exp_config, outdir, opt_var_name):
     )
 
 
-@cmor_cli.command()
-@click.option("-l", "--varlist",
-              type=str,
-              help=VARLIST_HELP,
-              required=False)
-@click.option("-r", "--table_config_dir",
-              type=str,
-              help="directory holding MIP tables to search for variables in var list",
-              required=True)
-@click.option('-v', "--opt_var_name",
-              type = str,
-              help=OPT_VAR_NAME_HELP,
-              required=False)
-def find(varlist, table_config_dir, opt_var_name):
-    '''
-    loop over json table files in config_dir and show which tables contain variables in var list/
-    the tool will also print what that table entry is expecting of that variable as well. if given
-    an opt_var_name in addition to varlist, only that variable name will be printed out.
-    accepts 3 arguments, two of the three required.
-    '''
-
-    # if opt_var_name specified, forget the list.
-    if opt_var_name is not None:
-        varlist=None
-
-    # custom arg requirement of "one of the two or both" in click should be implemented with
-    # logic before calling context.invoke( <thingy>, *args )
-    if opt_var_name is None and varlist is None:
-        raise ValueError('opt_var_name and varlist cannot both be None')
-
-    cmor_find_subtool(
-        json_var_list = varlist,
-        json_table_config_dir = table_config_dir,
-        opt_var_name = opt_var_name
-    )
-
-
 if __name__ == "__main__":
     cmor_cli()

From 508d57499c049dd3097e3eddddc7d2267c58932c Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 13:02:13 -0500
Subject: [PATCH 11/23] ok fine i am happy with that for now.

---
 fre/cmor/__init__.py | 4 ++++
 fre/cmor/frecmor.py  | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/fre/cmor/__init__.py b/fre/cmor/__init__.py
index e69de29b..a64139f5 100644
--- a/fre/cmor/__init__.py
+++ b/fre/cmor/__init__.py
@@ -0,0 +1,4 @@
+''' for fre.cmor imports '''
+from .cmor_mixer import cmor_run_subtool
+from .cmor_finder import cmor_find_subtool
+from .cmor_yamler import cmor_yaml_subtool
diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py
index 1e96e76d..46a2199f 100644
--- a/fre/cmor/frecmor.py
+++ b/fre/cmor/frecmor.py
@@ -2,9 +2,9 @@
 
 import click
 
-from .cmor_finder import cmor_find_subtool
-from .cmor_mixer import cmor_run_subtool
-from .cmor_yamler import cmor_yaml_subtool
+from . import cmor_find_subtool
+from . import cmor_run_subtool 
+from . import cmor_yaml_subtool
 
 OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \
                   "matching that variable name. I.e., this string help target local_vars, not " + \

From 8db16df8d4c14d2da0bc6db0a63acfa22bf2e7d8 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 13:17:54 -0500
Subject: [PATCH 12/23] update with cracking open the yaml file

---
 fre/cmor/cmor_mixer.py  | 11 ++++++-----
 fre/cmor/cmor_yamler.py | 25 +++++++++++++++++++++----
 fre/cmor/frecmor.py     |  2 +-
 3 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py
index 95377200..d33f32b2 100755
--- a/fre/cmor/cmor_mixer.py
+++ b/fre/cmor/cmor_mixer.py
@@ -669,11 +669,12 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
 
 
 
-def cmor_run_subtool( indir = None,
-                      json_var_list = None,
-                      json_table_config = None,
-                      json_exp_config = None ,
-                      outdir = None, opt_var_name = None
+def cmor_run_subtool( indir = None,             
+                      json_var_list = None,     
+                      json_table_config = None, 
+                      json_exp_config = None ,  
+                      outdir = None,            
+                      opt_var_name = None       
                       ):
     '''
     primary steering function for the cmor_mixer tool, i.e essentially main. Accepts six args:
diff --git a/fre/cmor/cmor_yamler.py b/fre/cmor/cmor_yamler.py
index fc1b0f90..f150cea0 100644
--- a/fre/cmor/cmor_yamler.py
+++ b/fre/cmor/cmor_yamler.py
@@ -1,18 +1,35 @@
+import logging
+fre_logger = logging.getLogger(__name__)
+
 import yaml
 
 from .cmor_mixer import cmor_run_subtool
 
-def cmor_yaml_subtool(yamlfile):
+def cmor_yaml_subtool(yamlfile = None):
     '''
     the thing that carries out the cmorization yamlerization
     '''
 
-    yaml_data = yaml.load(yamlfile, 'r') # check that
+    if yamlfile is None:
+        raise ValueError('(cmor_yaml_subtool) I need a yaml file.')
+    
+    fre_logger.info(f'reading: yamlfile = {yamlfile}')
+    with open(yamlfile,'r') as yamlfileobj:
+        yaml_data = yaml.safe_load(yamlfileobj)
+        fre_logger.info(f'yaml_data={yaml_data}')
+        
+
 
-    arg1 = yaml_data.access('KEY1' )
+    
+    #arg1 = yaml_data.access('KEY1' )
     
     cmor_run_subtool(
-        #        opt_var_name = arg1
+        indir = None,             
+        json_var_list = None,     
+        json_table_config = None, 
+        json_exp_config = None ,  
+        outdir = None,            
+        opt_var_name = None       
     )
 
     return
diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py
index 46a2199f..ddc46bd7 100644
--- a/fre/cmor/frecmor.py
+++ b/fre/cmor/frecmor.py
@@ -37,7 +37,7 @@ def yaml(yamlfile):
     if yamlfile is None:
         raise ValueError('I need a yamlfile!!!')
 
-    cmor_yamler_subtool(
+    cmor_yaml_subtool(
         yamlfile = yamlfile
     )
 

From 5638047fa3e4117539632cea63d59b105abd3c88 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 14:13:50 -0500
Subject: [PATCH 13/23] spot check- tests and basic yaml to slowly-transition
 to smth else.

---
 fre/cmor/cmor.yaml             |  5 +--
 fre/cmor/cmor_yamler.py        | 68 +++++++++++++++++++++++++++-------
 fre/tests/test_files/cmor.yaml | 37 ++++++++++++++++++
 fre/tests/test_fre_cmor_cli.py | 21 +++++++++++
 4 files changed, 114 insertions(+), 17 deletions(-)
 create mode 100644 fre/tests/test_files/cmor.yaml

diff --git a/fre/cmor/cmor.yaml b/fre/cmor/cmor.yaml
index 977cab82..015feea4 100644
--- a/fre/cmor/cmor.yaml
+++ b/fre/cmor/cmor.yaml
@@ -3,13 +3,12 @@
 
 # where we are
 cmor:
-  indir: ''
+  indir1: '' 
   json_var_list: ''
   json_table_config: ''
   json_exp_config: ''
   outdir: ''
-  opt_var_name: '' #[OPTIONAL SPECIAL-CASE ORIENTED USAGE]
-
+  opt_var_name: 
 
 
 
diff --git a/fre/cmor/cmor_yamler.py b/fre/cmor/cmor_yamler.py
index f150cea0..52c5bdd6 100644
--- a/fre/cmor/cmor_yamler.py
+++ b/fre/cmor/cmor_yamler.py
@@ -5,31 +5,71 @@
 
 from .cmor_mixer import cmor_run_subtool
 
-def cmor_yaml_subtool(yamlfile = None):
+def read_yaml_data(yamlfile = None):
     '''
-    the thing that carries out the cmorization yamlerization
     '''
-
     if yamlfile is None:
-        raise ValueError('(cmor_yaml_subtool) I need a yaml file.')
+        raise ValueError('(read_yaml_data) I need a yaml file.')
     
     fre_logger.info(f'reading: yamlfile = {yamlfile}')
+    yaml_data = None
+    
     with open(yamlfile,'r') as yamlfileobj:
-        yaml_data = yaml.safe_load(yamlfileobj)
+        all_yaml_data = yaml.safe_load(yamlfileobj)
         fre_logger.info(f'yaml_data={yaml_data}')
-        
 
+        try:
+            yaml_data=all_yaml_data['cmor']
+        except:
+            raise ValueError('(read_yaml_data) no data at cmor key in cmor yaml file!')
+            
+    if yaml_data is None:
+        raise ValueError(f'(read_yaml_data) could not read data from yamlfile={yamlfile}-yaml_data is None!')
 
+    return yaml_data
     
-    #arg1 = yaml_data.access('KEY1' )
-    
+
+def cmor_yaml_subtool(yamlfile = None):
+    '''
+    the thing that carries out the cmorization yamlerization
+    '''
+
+    yaml_data = read_yaml_data(yamlfile)
+        
+    # give reading a shot
+    indir = None              
+    json_var_list = None      
+    json_table_config = None  
+    json_exp_config = None    
+    outdir = None             
+
+    try:
+        fre_logger.info('reading key/values from yamlfile...')
+        indir = yaml_data['indir']
+        json_var_list = yaml_data['json_var_list']
+        json_table_config = yaml_data['json_table_config']
+        json_exp_config = yaml_data['json_exp_config']
+        outdir = yaml_data['outdir']
+    except:
+        raise ValueError(f'(cmor_yaml_subtool) {yamlfile} does not have all the required information.\n'
+                         f'(cmor_yaml_subtool) yaml_data=\n{yaml_data}'      )
+
+    # its ok if this one doesn't work out, not reqd anyway
+    opt_var_name = None                
+
+    try:
+        opt_var_name = yaml_data['opt_var_name']
+    except:
+        fre_logger.warning('could not read opt_var_name key/value. moving on.')
+
+    # showtime
     cmor_run_subtool(
-        indir = None,             
-        json_var_list = None,     
-        json_table_config = None, 
-        json_exp_config = None ,  
-        outdir = None,            
-        opt_var_name = None       
+        indir = indir,
+        json_var_list = json_var_list,
+        json_table_config = json_table_config,
+        json_exp_config = json_exp_config ,
+        outdir = outdir,
+        opt_var_name = opt_var_name
     )
 
     return
diff --git a/fre/tests/test_files/cmor.yaml b/fre/tests/test_files/cmor.yaml
new file mode 100644
index 00000000..5514aefa
--- /dev/null
+++ b/fre/tests/test_files/cmor.yaml
@@ -0,0 +1,37 @@
+
+
+
+# where we are
+cmor:
+  indir: 'fre/tests/test_files/ocean_sos_var_file' 
+  json_var_list: 'fre/tests/test_files/varlist' 
+  json_table_config: 'fre/tests/test_files/cmip6-cmor-tables/Tables/CMIP6_Omon.json' 
+  json_exp_config: 'fre/tests/test_files/CMOR_input_example.json'
+  outdir: 'fre/tests/test_files/ourdir'
+#  opt_var_name: 
+
+
+
+
+
+## from the test cases. 
+## Chris's desired end-state (best guess)
+#cmor:
+#  global:
+#    parent_experiment_name:
+#    xxxx
+#  Amon-tas-gr1:
+#    table: Amon
+#    variable: tas
+#    grid: gr1
+#    gfdl_comp: atmos
+#    gfdl_variable: t_ref
+#        '/archive/Eric.Stofferahn/ ' + \ #user
+#         'CMIP7/ESM4/DEV/' + \ #FRE stem
+#          'ESM4.5v01_om5b04_piC/' + \ #exp name
+#          'gfdl.ncrc5-intel23-prod-openmp/' + \ #platform
+#        'pp/' + \ #postprocess
+#        'land/' + \ # GFDL-component name
+#        'ts/' + \ # time-series
+#        'monthly/' + \ #frequency 
+#        '5yr/' # span of itme?
diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py
index 30a578e3..9bb0d36f 100644
--- a/fre/tests/test_fre_cmor_cli.py
+++ b/fre/tests/test_fre_cmor_cli.py
@@ -32,6 +32,27 @@ def test_cli_fre_cmor_opt_dne():
     result = runner.invoke(fre.fre, args=["cmor", "optionDNE"])
     assert result.exit_code == 2
 
+# fre cmor yaml
+def test_cli_fre_cmor_yaml():
+    ''' fre cmor yaml '''
+    result = runner.invoke(fre.fre, args=["cmor", "yaml"])
+    assert result.exit_code == 2
+
+def test_cli_fre_cmor_yaml_help():
+    ''' fre cmor yaml --help '''
+    result = runner.invoke(fre.fre, args=["cmor", "yaml", "--help"])
+    assert result.exit_code == 0
+
+def test_cli_fre_cmor_yaml_opt_dne():
+    ''' fre cmor yaml optionDNE '''
+    result = runner.invoke(fre.fre, args=["cmor", "yaml", "optionDNE"])
+    assert result.exit_code == 2
+
+def test_cli_fre_cmor_yaml_case1():
+    ''' fre cmor yaml -y '''
+    result = runner.invoke(fre.fre, args=["cmor", "yaml", "-y", f"{rootdir}/cmor.yaml"])
+    assert result.exit_code == 0
+
 # fre cmor run
 def test_cli_fre_cmor_run():
     ''' fre cmor run '''

From 78767d66078e8624935833dc16cce3c4ddbd3d6a Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 14:22:33 -0500
Subject: [PATCH 14/23] proper unit test- plugged in. naive yaml case works
 fine.

---
 fre/tests/test_files/cmor.yaml |  2 +-
 fre/tests/test_fre_cmor_cli.py | 97 +++++++++++++++++++++-------------
 2 files changed, 60 insertions(+), 39 deletions(-)

diff --git a/fre/tests/test_files/cmor.yaml b/fre/tests/test_files/cmor.yaml
index 5514aefa..a0aa9ed8 100644
--- a/fre/tests/test_files/cmor.yaml
+++ b/fre/tests/test_files/cmor.yaml
@@ -7,7 +7,7 @@ cmor:
   json_var_list: 'fre/tests/test_files/varlist' 
   json_table_config: 'fre/tests/test_files/cmip6-cmor-tables/Tables/CMIP6_Omon.json' 
   json_exp_config: 'fre/tests/test_files/CMOR_input_example.json'
-  outdir: 'fre/tests/test_files/ourdir'
+  outdir: 'fre/tests/test_files/outdir'
 #  opt_var_name: 
 
 
diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py
index 9bb0d36f..5bee5976 100644
--- a/fre/tests/test_fre_cmor_cli.py
+++ b/fre/tests/test_fre_cmor_cli.py
@@ -14,7 +14,31 @@
 runner = CliRunner()
 
 # where are we? we're running pytest from the base directory of this repo
-rootdir = 'fre/tests/test_files'
+ROOTDIR = 'fre/tests/test_files'
+
+# these unit tests should be more about the cli, rather than the workload
+YYYYMMDD=date.today().strftime('%Y%m%d')
+
+copied_nc_filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sosV2.nc'
+full_copied_nc_filepath = f'{ROOTDIR}/ocean_sos_var_file/{copied_nc_filename}'
+original_nc_file = f'{ROOTDIR}/ocean_sos_var_file/reduced_ocean_monthly_1x1deg.199307-199308.sos.nc'
+
+def test_setup_test_files(capfd):
+    "set-up test: copy and rename NetCDF file created in test_fre_cmor_run_subtool.py"
+
+    assert Path(original_nc_file).exists()
+
+    if Path(full_copied_nc_filepath).exists():
+        Path(full_copied_nc_filepath).unlink()
+    assert not Path(full_copied_nc_filepath).exists()
+
+    shutil.copy(Path(original_nc_file), Path(full_copied_nc_filepath))
+
+    assert (Path(full_copied_nc_filepath).exists())
+
+    out, err = capfd.readouterr()
+
+
 
 # fre cmor
 def test_cli_fre_cmor():
@@ -50,8 +74,29 @@ def test_cli_fre_cmor_yaml_opt_dne():
 
 def test_cli_fre_cmor_yaml_case1():
     ''' fre cmor yaml -y '''
-    result = runner.invoke(fre.fre, args=["cmor", "yaml", "-y", f"{rootdir}/cmor.yaml"])
-    assert result.exit_code == 0
+    
+    # FYI
+    indir = f'{ROOTDIR}/ocean_sos_var_file'
+    filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sos.nc' # unneeded, this is mostly for reference
+    full_inputfile=f"{indir}/{filename}"
+    
+    # determined by cmor_run_subtool
+    outdir = f'{ROOTDIR}/outdir'
+    cmor_creates_dir = \
+        'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Omon/sos/gn'
+    full_outputdir = \
+        f"{outdir}/{cmor_creates_dir}/v{YYYYMMDD}" # yay no more 'fre' where it shouldnt be
+    full_outputfile = \
+        f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199308.nc"
+
+
+    result = runner.invoke(fre.fre, args=["cmor", "yaml", "-y", f"{ROOTDIR}/cmor.yaml"])
+
+
+    assert all ( [ result.exit_code == 0,
+                   Path(full_outputfile).exists(),
+                   Path(full_inputfile).exists() ] )
+
 
 # fre cmor run
 def test_cli_fre_cmor_run():
@@ -69,36 +114,15 @@ def test_cli_fre_cmor_run_opt_dne():
     result = runner.invoke(fre.fre, args=["cmor", "run", "optionDNE"])
     assert result.exit_code == 2
 
-copied_nc_filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sosV2.nc'
-full_copied_nc_filepath = f'{rootdir}/ocean_sos_var_file/{copied_nc_filename}'
-original_nc_file = f'{rootdir}/ocean_sos_var_file/reduced_ocean_monthly_1x1deg.199307-199308.sos.nc'
-
-def test_setup_test_files(capfd):
-    "set-up test: copy and rename NetCDF file created in test_fre_cmor_run_subtool.py"
-
-    assert Path(original_nc_file).exists()
-
-    if Path(full_copied_nc_filepath).exists():
-        Path(full_copied_nc_filepath).unlink()
-    assert not Path(full_copied_nc_filepath).exists()
-
-    shutil.copy(Path(original_nc_file), Path(full_copied_nc_filepath))
-
-    assert (Path(full_copied_nc_filepath).exists())
-
-    out, err = capfd.readouterr()
-
-# these unit tests should be more about the cli, rather than the workload
-YYYYMMDD=date.today().strftime('%Y%m%d')
 def test_cli_fre_cmor_run_case1(capfd):
     ''' fre cmor run, test-use case '''
 
     # explicit inputs to tool
-    indir = f'{rootdir}/ocean_sos_var_file/'
-    varlist = f'{rootdir}/varlist'
-    table_config = f'{rootdir}/cmip6-cmor-tables/Tables/CMIP6_Omon.json'
-    exp_config = f'{rootdir}/CMOR_input_example.json'
-    outdir = f'{rootdir}/outdir'
+    indir = f'{ROOTDIR}/ocean_sos_var_file/'
+    varlist = f'{ROOTDIR}/varlist'
+    table_config = f'{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_Omon.json'
+    exp_config = f'{ROOTDIR}/CMOR_input_example.json'
+    outdir = f'{ROOTDIR}/outdir'
 
     # determined by cmor_run_subtool
     cmor_creates_dir = \
@@ -132,16 +156,13 @@ def test_cli_fre_cmor_run_case1(capfd):
 
 def test_cli_fre_cmor_run_case2(capfd):
     ''' fre cmor run, test-use case '''
-
-    # where are we? we're running pytest from the base directory of this repo
-    rootdir = 'fre/tests/test_files'
-
+    
     # explicit inputs to tool
-    indir = f'{rootdir}/ocean_sos_var_file'
-    varlist = f'{rootdir}/varlist_local_target_vars_differ'
-    table_config = f'{rootdir}/cmip6-cmor-tables/Tables/CMIP6_Omon.json'
-    exp_config = f'{rootdir}/CMOR_input_example.json'
-    outdir = f'{rootdir}/outdir'
+    indir = f'{ROOTDIR}/ocean_sos_var_file'
+    varlist = f'{ROOTDIR}/varlist_local_target_vars_differ'
+    table_config = f'{ROOTDIR}/cmip6-cmor-tables/Tables/CMIP6_Omon.json'
+    exp_config = f'{ROOTDIR}/CMOR_input_example.json'
+    outdir = f'{ROOTDIR}/outdir'
 
     # determined by cmor_run_subtool
     cmor_creates_dir = \

From b7df165100e7834ce2ce8d3523df33e453a230c6 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 31 Jan 2025 17:27:30 -0500
Subject: [PATCH 15/23] catch up and push- show me what i am working with

---
 .gitignore                     |  6 ------
 fre/cmor/cmor.yaml             | 29 ++++++++++++++++-------------
 fre/cmor/cmor_mixer.py         | 12 ++++++------
 fre/cmor/cmor_yamler.py        |  1 +
 fre/tests/test_files/cmor.yaml | 16 +++++++++++-----
 5 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/.gitignore b/.gitignore
index b2cf9b85..5ef15fd6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -55,12 +55,6 @@ coverage.xml
 .pytest_cache/
 cover/
 
-<<<<<<< HEAD
-
-=======
-
->>>>>>> main
-
 # Translations
 *.mo
 *.pot
diff --git a/fre/cmor/cmor.yaml b/fre/cmor/cmor.yaml
index 015feea4..3cdd79a3 100644
--- a/fre/cmor/cmor.yaml
+++ b/fre/cmor/cmor.yaml
@@ -3,7 +3,10 @@
 
 # where we are
 cmor:
-  indir1: '' 
+  global:
+    table_dir: ''
+    
+  indir: ''
   json_var_list: ''
   json_table_config: ''
   json_exp_config: ''
@@ -17,21 +20,21 @@ cmor:
 ## from the test cases. 
 ## Chris's desired end-state (best guess)
 #cmor:
-#  global:
-#    parent_experiment_name:
-#    xxxx
+##  global:
+##    parent_experiment_name:
+##    xxxx
 #  Amon-tas-gr1:
 #    table: Amon
 #    variable: tas
 #    grid: gr1
 #    gfdl_comp: atmos
 #    gfdl_variable: t_ref
-#        '/archive/Eric.Stofferahn/ ' + \ #user
-#         'CMIP7/ESM4/DEV/' + \ #FRE stem
-#          'ESM4.5v01_om5b04_piC/' + \ #exp name
-#          'gfdl.ncrc5-intel23-prod-openmp/' + \ #platform
-#        'pp/' + \ #postprocess
-#        'land/' + \ # GFDL-component name
-#        'ts/' + \ # time-series
-#        'monthly/' + \ #frequency 
-#        '5yr/' # span of itme?
+#    user: '/archive/Eric.Stofferahn/'
+#    FRE_STEM: 'CMIP7/ESM4/DEV/' 
+#    expname: 'ESM4.5v01_om5b04_piC/' 
+#    platform: 'gfdl.ncrc5-intel23-prod-openmp/'
+#    pp: True #'pp'
+#    gfdl_component: 'land'
+#    ts_or_av: 'ts' #ts t-series, av t-averages
+#    freq: 'monthly' # freq of data in a file
+#    delta_t: '5yr' # span of time in one file, aka chunk?
diff --git a/fre/cmor/cmor_mixer.py b/fre/cmor/cmor_mixer.py
index d33f32b2..0d33a6a1 100755
--- a/fre/cmor/cmor_mixer.py
+++ b/fre/cmor/cmor_mixer.py
@@ -669,12 +669,12 @@ def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
 
 
 
-def cmor_run_subtool( indir = None,             
-                      json_var_list = None,     
-                      json_table_config = None, 
-                      json_exp_config = None ,  
-                      outdir = None,            
-                      opt_var_name = None       
+def cmor_run_subtool( indir = None,
+                      json_var_list = None,
+                      json_table_config = None,
+                      json_exp_config = None ,
+                      outdir = None,
+                      opt_var_name = None
                       ):
     '''
     primary steering function for the cmor_mixer tool, i.e essentially main. Accepts six args:
diff --git a/fre/cmor/cmor_yamler.py b/fre/cmor/cmor_yamler.py
index 52c5bdd6..efdc5740 100644
--- a/fre/cmor/cmor_yamler.py
+++ b/fre/cmor/cmor_yamler.py
@@ -7,6 +7,7 @@
 
 def read_yaml_data(yamlfile = None):
     '''
+    from string, open a file object and load/return yaml data under key 'cmor'
     '''
     if yamlfile is None:
         raise ValueError('(read_yaml_data) I need a yaml file.')
diff --git a/fre/tests/test_files/cmor.yaml b/fre/tests/test_files/cmor.yaml
index a0aa9ed8..c085fb41 100644
--- a/fre/tests/test_files/cmor.yaml
+++ b/fre/tests/test_files/cmor.yaml
@@ -3,11 +3,17 @@
 
 # where we are
 cmor:
-  indir: 'fre/tests/test_files/ocean_sos_var_file' 
-  json_var_list: 'fre/tests/test_files/varlist' 
-  json_table_config: 'fre/tests/test_files/cmip6-cmor-tables/Tables/CMIP6_Omon.json' 
-  json_exp_config: 'fre/tests/test_files/CMOR_input_example.json'
-  outdir: 'fre/tests/test_files/outdir'
+  global:
+    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
+
+  Omon-ts-gr1:
+    table: 'Omon'
+    grid: 'gr1'
+    indir: 'fre/tests/test_files/ocean_sos_var_file' 
+    json_var_list: 'fre/tests/test_files/varlist' 
+    json_table_config: !join[ *table_dir ,'/', 'CMIP6_', *table, '.json'  ]
+    json_exp_config: 'fre/tests/test_files/CMOR_input_example.json'
+    outdir: 'fre/tests/test_files/outdir'
 #  opt_var_name: 
 
 

From df0899d2d000864c9574e41937ccd1f43ac45f9c Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Tue, 11 Feb 2025 10:52:16 -0500
Subject: [PATCH 16/23] test yaml deets to test routine

---
 fre/tests/test_fre_cmor_cli.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py
index 5bee5976..ca7eaf2d 100644
--- a/fre/tests/test_fre_cmor_cli.py
+++ b/fre/tests/test_fre_cmor_cli.py
@@ -72,6 +72,7 @@ def test_cli_fre_cmor_yaml_opt_dne():
     result = runner.invoke(fre.fre, args=["cmor", "yaml", "optionDNE"])
     assert result.exit_code == 2
 
+TEST_YAML_PATH=f"{ROOTDIR}/cmor.yaml"
 def test_cli_fre_cmor_yaml_case1():
     ''' fre cmor yaml -y '''
     
@@ -90,10 +91,11 @@ def test_cli_fre_cmor_yaml_case1():
         f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199308.nc"
 
 
-    result = runner.invoke(fre.fre, args=["cmor", "yaml", "-y", f"{ROOTDIR}/cmor.yaml"])
+    result = runner.invoke(fre.fre, args=["cmor", "yaml", "-y", TEST_YAML_PATH])
 
 
     assert all ( [ result.exit_code == 0,
+                   Path(TEST_YAML_PATH).exists(), 
                    Path(full_outputfile).exists(),
                    Path(full_inputfile).exists() ] )
 

From 290bacfd723f701a0b8c9d9ed1839c94d5ca0102 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Tue, 11 Feb 2025 10:52:53 -0500
Subject: [PATCH 17/23] add quick/dirty scratch work script im using to check
 on yaml things

---
 yaml_check.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 yaml_check.py

diff --git a/yaml_check.py b/yaml_check.py
new file mode 100644
index 00000000..e21c5d2e
--- /dev/null
+++ b/yaml_check.py
@@ -0,0 +1,13 @@
+import yaml
+
+from fre.yamltools.combine_yamls import join_constructor as jc
+yaml.add_constructor("!join", jc)
+
+file_obj=open("fre/tests/test_files/cmor.yaml",
+              "r+", encoding='utf-8')
+
+
+#yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
+yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
+
+print(yaml_data)

From 22aa889f29d630e64f9ed4d109b693c516e4e281 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Tue, 11 Feb 2025 11:32:52 -0500
Subject: [PATCH 18/23] oh, yaml doesn't hate whitespace quite as much as i
 thought, have yaml_check.py use pprint

---
 fre/tests/test_files/cmor.yaml | 152 +++++++++++++++++++++++++++------
 yaml_check.py                  |   5 +-
 2 files changed, 132 insertions(+), 25 deletions(-)

diff --git a/fre/tests/test_files/cmor.yaml b/fre/tests/test_files/cmor.yaml
index c085fb41..7de37638 100644
--- a/fre/tests/test_files/cmor.yaml
+++ b/fre/tests/test_files/cmor.yaml
@@ -1,27 +1,131 @@
+cmor:
+
+  globals:
+    table_dir:
+      &table_dir "fre/tests/test_files/cmip6-cmor-tables/Tables"
+    exp_config_dir:
+      &exp_config_dir "fre/tests/test_files"
+
+  targets:
+    Omon-ts-gr1:
+      json_table_config:
+        !join [ *table_dir, "/", "CMIP6_Omon.json" ]
+      json_exp_config:
+        !join [ *exp_config_dir, "/", "CMOR_input_example.json" ]
+      indir:
+        "fre/tests/test_files/ocean_sos_var_file"
+      json_var_list:
+        "fre/tests/test_files/varlist"
+      outdir:
+        "fre/tests/test_files/outdir"
+      opt_var_name:
+
+
+
+
+
+
+
+
+
+#cmor:
+#  globals: &globals
+#    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
+#  targets:
+#    Omon-ts-gr1:
+#      <<: *globals
+#      json_table_config: !join [ <table_dir>, "/", "CMIP6_Omon.json" ]
+#
+#
+
+
+
+
+
+
+
+
+
+
+
+#cmor:
+#  globals: &globals
+#    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
+#
+#  targets:
+#    <<: *globals
+#    Omon-ts-gr1:
+#      json_table_config: !join [<table_dir>, "/", "CMIP6_Omon.json"]
+#      table: "Omon"
+#      grid: "gr1"
+#      indir: "fre/tests/test_files/ocean_sos_var_file"
+#      json_var_list: "fre/tests/test_files/varlist"
+#      json_exp_config: "fre/tests/test_files/CMOR_input_example.json"
+#      outdir: "fre/tests/test_files/outdir"
+#
+#
+#
+#
+#
+#
+#
+#
+
+
+
+
 
 
 
-# where we are
-cmor:
-  global:
-    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
-
-  Omon-ts-gr1:
-    table: 'Omon'
-    grid: 'gr1'
-    indir: 'fre/tests/test_files/ocean_sos_var_file' 
-    json_var_list: 'fre/tests/test_files/varlist' 
-    json_table_config: !join[ *table_dir ,'/', 'CMIP6_', *table, '.json'  ]
-    json_exp_config: 'fre/tests/test_files/CMOR_input_example.json'
-    outdir: 'fre/tests/test_files/outdir'
-#  opt_var_name: 
 
 
 
 
 
+
+#  - "Omon"      
+#    grid : "gr1"
+#    indir : "fre/tests/test_files/ocean_sos_var_file" 
+#    json_var_list : "fre/tests/test_files/varlist" 
+#    json_exp_config : "fre/tests/test_files/CMOR_input_example.json"
+#    outdir : "fre/tests/test_files/outdir"
+
+
+
+
+
+
+
+
+
+
+
+#    BAZ:
+#      smth: <table_dir>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#
+#
+#
+#    - &table "Omon"
+#  opt_var_name: 
 ## from the test cases. 
-## Chris's desired end-state (best guess)
+## Chris"s desired end-state (best guess)
 #cmor:
 #  global:
 #    parent_experiment_name:
@@ -32,12 +136,12 @@ cmor:
 #    grid: gr1
 #    gfdl_comp: atmos
 #    gfdl_variable: t_ref
-#        '/archive/Eric.Stofferahn/ ' + \ #user
-#         'CMIP7/ESM4/DEV/' + \ #FRE stem
-#          'ESM4.5v01_om5b04_piC/' + \ #exp name
-#          'gfdl.ncrc5-intel23-prod-openmp/' + \ #platform
-#        'pp/' + \ #postprocess
-#        'land/' + \ # GFDL-component name
-#        'ts/' + \ # time-series
-#        'monthly/' + \ #frequency 
-#        '5yr/' # span of itme?
+#        "/archive/Eric.Stofferahn/ " + \ #user
+#         "CMIP7/ESM4/DEV/" + \ #FRE stem
+#          "ESM4.5v01_om5b04_piC/" + \ #exp name
+#          "gfdl.ncrc5-intel23-prod-openmp/" + \ #platform
+#        "pp/" + \ #postprocess
+#        "land/" + \ # GFDL-component name
+#        "ts/" + \ # time-series
+#        "monthly/" + \ #frequency 
+#        "5yr/" # span of itme?
diff --git a/yaml_check.py b/yaml_check.py
index e21c5d2e..fee803cd 100644
--- a/yaml_check.py
+++ b/yaml_check.py
@@ -10,4 +10,7 @@
 #yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
 yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
 
-print(yaml_data)
+import pprint
+pp = pprint.PrettyPrinter(indent=2)
+
+pp.pprint(yaml_data['cmor'])

From 2141987e1042287204e8aa25feaa1da804aba41c Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Tue, 11 Feb 2025 13:03:12 -0500
Subject: [PATCH 19/23] bump cmor cmip6 table submodule commit

---
 fre/tests/test_files/cmip6-cmor-tables | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fre/tests/test_files/cmip6-cmor-tables b/fre/tests/test_files/cmip6-cmor-tables
index 6a30acae..a46dcbf1 160000
--- a/fre/tests/test_files/cmip6-cmor-tables
+++ b/fre/tests/test_files/cmip6-cmor-tables
@@ -1 +1 @@
-Subproject commit 6a30acaea9b63fdecbe08894d7e50ef36879a993
+Subproject commit a46dcbf17ec5e11af23dc2d55107f5e52afbcade

From 45887441a6b2bebb2768de7869b2722f019bd7ae Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Tue, 11 Feb 2025 13:03:21 -0500
Subject: [PATCH 20/23] tweak git ignore

---
 .gitignore | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5ef15fd6..afadde6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -158,7 +158,7 @@ cython_debug/
 
 # other
 *~
-*\#
+*\#*
 
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
@@ -173,29 +173,27 @@ cython_debug/
 
 
 ##### GOTTA CLEAN THIS UP
+
 # files pulled/created via testing
-fre/tests/test_files/outdirfre/
-fre/pp/tests/configure_yaml_out/
-fre/yamltools/tests/combine_yamls_out/
-fre/app/generate_time_averages/tests/time_avg_test_files/
+tmp/
 
-# fremake generated files that should be ignored
-combined-null_model.yaml
-combined-am5.yaml
-tmp
-makefile_out
+fre/tests/fremake_out
 fre/tests/test_files/ocean_sos_var_file/*.nc
+fre/tests/test_files/outdirfre/
 fre/tests/test_files/outdir/
 
+fre/app/generate_time_averages/tests/time_avg_test_files/
+fre/app/regrid_xy/tests/*/
 
+#fre/make/tests/null_example/combined-null_model.yaml
+fre/make/tests/*/
 
-# files pulled/created via testing
-#fre/tests/test_files/cmip6-cmor-tables/
-fre/tests/test_files/outdirfre/
 fre/pp/tests/configure_yaml_out/
+
 fre/yamltools/tests/combine_yamls_out/
-fre/app/generate_time_averages/tests/time_avg_test_files/
-fre/make/tests/null_example/combined-null_model.yaml
-fre/make/tests/makefile_out
-fre/make/tests/compile_out
-fre/tests/fremake_out
\ No newline at end of file
+
+
+# fremake generated files that should be ignored
+#combined-null_model.yaml
+#combined-am5.yaml
+#makefile_out
\ No newline at end of file

From e0a157b0040048c085dbef0033203bda1ac1f1b8 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Wed, 12 Feb 2025 15:06:57 -0500
Subject: [PATCH 21/23] lets go with something like this for the yaml....
 coalescing... tweak check script- maybe it hsould be a yamltool

---
 fre/tests/test_files/cmor.yaml | 130 +--------------------------------
 yaml_check.py                  |  31 ++++++--
 2 files changed, 28 insertions(+), 133 deletions(-)

diff --git a/fre/tests/test_files/cmor.yaml b/fre/tests/test_files/cmor.yaml
index 7de37638..c1508ec9 100644
--- a/fre/tests/test_files/cmor.yaml
+++ b/fre/tests/test_files/cmor.yaml
@@ -1,10 +1,13 @@
+
 cmor:
 
-  globals:
+  directories:
+    <<: *shared_directories
     table_dir:
       &table_dir "fre/tests/test_files/cmip6-cmor-tables/Tables"
     exp_config_dir:
       &exp_config_dir "fre/tests/test_files"
+      
 
   targets:
     Omon-ts-gr1:
@@ -20,128 +23,3 @@ cmor:
         "fre/tests/test_files/outdir"
       opt_var_name:
 
-
-
-
-
-
-
-
-
-#cmor:
-#  globals: &globals
-#    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
-#  targets:
-#    Omon-ts-gr1:
-#      <<: *globals
-#      json_table_config: !join [ <table_dir>, "/", "CMIP6_Omon.json" ]
-#
-#
-
-
-
-
-
-
-
-
-
-
-
-#cmor:
-#  globals: &globals
-#    table_dir: "fre/tests/test_files/cmip6-cmor-tables/Tables"
-#
-#  targets:
-#    <<: *globals
-#    Omon-ts-gr1:
-#      json_table_config: !join [<table_dir>, "/", "CMIP6_Omon.json"]
-#      table: "Omon"
-#      grid: "gr1"
-#      indir: "fre/tests/test_files/ocean_sos_var_file"
-#      json_var_list: "fre/tests/test_files/varlist"
-#      json_exp_config: "fre/tests/test_files/CMOR_input_example.json"
-#      outdir: "fre/tests/test_files/outdir"
-#
-#
-#
-#
-#
-#
-#
-#
-
-
-
-
-
-
-
-
-
-
-
-
-
-#  - "Omon"      
-#    grid : "gr1"
-#    indir : "fre/tests/test_files/ocean_sos_var_file" 
-#    json_var_list : "fre/tests/test_files/varlist" 
-#    json_exp_config : "fre/tests/test_files/CMOR_input_example.json"
-#    outdir : "fre/tests/test_files/outdir"
-
-
-
-
-
-
-
-
-
-
-
-#    BAZ:
-#      smth: <table_dir>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-#
-#
-#
-#    - &table "Omon"
-#  opt_var_name: 
-## from the test cases. 
-## Chris"s desired end-state (best guess)
-#cmor:
-#  global:
-#    parent_experiment_name:
-#    xxxx
-#  Amon-tas-gr1:
-#    table: Amon
-#    variable: tas
-#    grid: gr1
-#    gfdl_comp: atmos
-#    gfdl_variable: t_ref
-#        "/archive/Eric.Stofferahn/ " + \ #user
-#         "CMIP7/ESM4/DEV/" + \ #FRE stem
-#          "ESM4.5v01_om5b04_piC/" + \ #exp name
-#          "gfdl.ncrc5-intel23-prod-openmp/" + \ #platform
-#        "pp/" + \ #postprocess
-#        "land/" + \ # GFDL-component name
-#        "ts/" + \ # time-series
-#        "monthly/" + \ #frequency 
-#        "5yr/" # span of itme?
diff --git a/yaml_check.py b/yaml_check.py
index fee803cd..3f2cd8cc 100644
--- a/yaml_check.py
+++ b/yaml_check.py
@@ -1,16 +1,33 @@
+import click
+
 import yaml
 
 from fre.yamltools.combine_yamls import join_constructor as jc
 yaml.add_constructor("!join", jc)
 
-file_obj=open("fre/tests/test_files/cmor.yaml",
-              "r+", encoding='utf-8')
 
 
-#yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
-yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
 
-import pprint
-pp = pprint.PrettyPrinter(indent=2)
 
-pp.pprint(yaml_data['cmor'])
+
+
+def pprint_yaml():
+    file_obj=open("fre/tests/test_files/cmor.yaml",
+                  "r+", encoding='utf-8')
+    
+    
+    #yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
+    yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
+    
+    import pprint
+    pp = pprint.PrettyPrinter(indent=2)
+    
+    pp.pprint(yaml_data['cmor'])
+
+
+    
+def main():
+    pprint_yaml()
+
+if __name__ == "__main__":
+    main()

From eab37a99b3b2a61890fea784ccd6f4f710374c3c Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Thu, 13 Feb 2025 16:23:12 -0500
Subject: [PATCH 22/23] yamlcheck edits

---
 yaml_check.py | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/yaml_check.py b/yaml_check.py
index 3f2cd8cc..81603eb2 100644
--- a/yaml_check.py
+++ b/yaml_check.py
@@ -11,23 +11,25 @@
 
 
 
-def pprint_yaml():
-    file_obj=open("fre/tests/test_files/cmor.yaml",
-                  "r+", encoding='utf-8')
-    
-    
-    #yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
-    yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
-    
-    import pprint
-    pp = pprint.PrettyPrinter(indent=2)
-    
-    pp.pprint(yaml_data['cmor'])
-
-
-    
-def main():
-    pprint_yaml()
-
-if __name__ == "__main__":
-    main()
+
+#file_obj=open("fre/tests/test_files/cmor.yaml",
+#file_obj=open("fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml",
+file_obj=open("fre/yamltools/tests/AM5_example/am5.yaml",
+              "r+", encoding='utf-8')
+
+
+#yaml_data=yaml.safe_load(file_obj) #Loader=yaml.Loader)
+yaml_data=yaml.load(file_obj, Loader=yaml.Loader)
+
+import pprint
+pp = pprint.PrettyPrinter(indent=2)
+
+pp.pprint(yaml_data)
+
+
+
+#def main():
+#    pprint_yaml()
+#
+#if __name__ == "__main__":
+#    main()

From d476680dccb2078977bb8559cf764c903bc86369 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Thu, 13 Feb 2025 16:23:17 -0500
Subject: [PATCH 23/23] rename yaml

---
 .../tests/AM5_example/cmor_yamls/cmor.am5.yaml}                   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename fre/{tests/test_files/cmor.yaml => yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml} (100%)

diff --git a/fre/tests/test_files/cmor.yaml b/fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml
similarity index 100%
rename from fre/tests/test_files/cmor.yaml
rename to fre/yamltools/tests/AM5_example/cmor_yamls/cmor.am5.yaml