Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable bias correction for aerosol DA #3189

Draft
wants to merge 9 commits into
base: develop
Choose a base branch
from
Draft

Enable bias correction for aerosol DA #3189

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d27c9a4
handle aerosol bias correction files
Dec 20, 2024
5c7b192
fix code norm check
Dec 20, 2024
3975553
Merge branch 'NOAA-EMC:develop' into feature/aero-varbc
ypwang19 Dec 31, 2024
cccfe73
change diag file suffix
Jan 2, 2025
ff4821d
Merge branch 'develop' into feature/aero-varbc
ypwang19 Jan 3, 2025
a40982a
update hash to new GDASApp develop
Jan 3, 2025
1d0ed7a
update genb parameters for higher resolution
Jan 6, 2025
addd27d
Merge branch 'develop' into feature/aero-varbc
CoryMartin-NOAA Jan 6, 2025
533130c
Update jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE
ypwang19 Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \

RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \
COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL
COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL \
COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL
ypwang19 marked this conversation as resolved.
Show resolved Hide resolved

###############################################################
# Run relevant script
Expand Down
10 changes: 5 additions & 5 deletions parm/config/gfs/config.aeroanlgenb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,17 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero_bmat_jedi_config.yaml.j2"
export JCB_BASE_YAML="${PARMgfs}/gdas/aero/jcb-base.yaml.j2"
export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2"
export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2"
export aero_diffusion_iter=10
export aero_diffusion_horiz_len=2500e3
export aero_diffusion_fixed_val=1.0
export aero_diffusion_iter=200
export aero_diffusion_horiz_len=300e3
export aero_diffusion_fixed_val=10.0
export npx_clim_b=97
export npy_clim_b=97
export aero_diagb_weight=0.9
export aero_staticb_rescaling_factor=2.0
export aero_diagb_rescale=20.0
export aero_diagb_rescale=100.0
export aero_diagb_n_halo=4
export aero_diagb_n_neighbors=16
export aero_diagb_smooth_horiz_iter=0
export aero_diagb_smooth_horiz_iter=200
export aero_diagb_smooth_vert_iter=0

echo "END: config.aeroanlgenb"
2 changes: 1 addition & 1 deletion sorc/gdas.cd
Open in desktop
Submodule gdas.cd updated 47 files
+24 −0 .github/pull_request_template.md
+6 −0 ci/ci_tests.sh
+123 −47 ci/driver.sh
+0 −169 ci/gw_driver.sh
+1 −0 ci/hera.sh
+13 −0 ci/hercules.sh
+3 −1 ci/orion.sh
+52 −10 ci/run_ci.sh
+0 −80 ci/run_gw_ci.sh
+24 −10 ci/stable_driver.sh
+3 −9 parm/aero/jcb-base.yaml.j2
+6 −10 parm/io/fv3jedi_fieldmetadata_history.yaml
+1 −1 parm/jcb-algorithms
+1 −1 parm/jcb-gdas
+10 −2 parm/soca/marine-jcb-base.yaml
+1 −1 prototypes/gen_prototype.sh
+1 −1 sorc/da-utils
+1 −1 sorc/fv3-jedi
+1 −1 sorc/ioda
+1 −1 sorc/iodaconv
+1 −1 sorc/oops
+1 −1 sorc/saber
+1 −1 sorc/soca
+1 −1 sorc/ufo
+1 −1 sorc/vader
+3 −3 test/atm/global-workflow/CMakeLists.txt
+5 −4 test/atm/global-workflow/jjob_ens_init_split.sh
+8 −4 test/gw-ci/CMakeLists.txt
+17 −17 test/testreference/C96C48_ufs_hybatmDA_3dvar-fv3inc.ref
+34 −34 test/testreference/C96C48_ufs_hybatmDA_3dvar.ref
+4 −4 test/testreference/C96C48_ufs_hybatmDA_lgetkf_observer.ref
+8 −8 test/testreference/C96C48_ufs_hybatmDA_lgetkf_solver.ref
+30 −33 test/testreference/atm_jjob_3dvar.ref
+65 −65 test/testreference/atm_jjob_lgetkf.ref
+29 −29 test/testreference/atm_jjob_lgetkf_observer.ref
+48 −48 test/testreference/atm_jjob_lgetkf_solver.ref
+1 −1 utils/chem/chem_diagb.h
+1 −1 utils/fv3jedi/fv3jedi_fv3inc.h
+1 −1 utils/ioda_example/gdas_meanioda.h
+1 −1 utils/land/land_ensrecenter.h
+1 −1 utils/obsproc/applications/gdas_obsprovider2ioda.h
+4 −4 utils/soca/gdas_ens_handler.h
+14 −8 utils/soca/gdas_incr_handler.h
+26 −18 utils/soca/gdas_postprocincr.h
+1 −1 utils/soca/gdas_soca_diagb.h
+1 −1 utils/soca/gdas_socahybridweights.h
+1 −1 utils/soca/gdassoca_obsstats.h
38 changes: 25 additions & 13 deletions ush/python/pygfs/task/aero_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,17 +105,18 @@ def initialize(self) -> None:
logger.debug(f"Observation files:\n{pformat(obs_dict)}")

# # stage bias corrections
# logger.info(f"Staging list of bias correction files")
# bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging')
# if bias_dict['copy'] is None:
# logger.info(f"No bias correction files to stage")
# else:
# bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy'])
# FileHandler(bias_dict).sync()
# logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")

# # extract bias corrections
# Jedi.extract_tar_from_filehandler_dict(bias_dict)
logger.info(f"Staging list of bias correction files")
bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging')
logger.debug(bias_dict)
if bias_dict['copy'] is None:
logger.info(f"No bias correction files to stage")
else:
bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy'])
FileHandler(bias_dict).sync()
logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")

# extract bias corrections
Jedi.extract_tar_from_filehandler_dict(bias_dict)

# stage CRTM fix files
logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")
Expand Down Expand Up @@ -170,7 +171,7 @@ def finalize(self) -> None:
aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat")

# get list of diag files to put in tarball
diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4'))
diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc'))

# gzip the files first
for diagfile in diags:
Expand All @@ -183,13 +184,24 @@ def finalize(self) -> None:
self._add_fms_cube_sphere_increments()

# tar up bias correction files
# NOTE TODO
bfile = f"{self.task_config.APREFIX}aero_varbc_params.tar"
aertar = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, bfile)

# get lists of aerosol bias correction files to add to tarball
satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc'))

# copy files back to COM
logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}")
aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config)
FileHandler(aero_var_final_list).sync()

# tar aerosol bias correction files to ROTDIR
logger.info(f"Creating aerosol bias correction tar file {aertar}")
with tarfile.open(aertar, 'w') as aerbcor:
for satfile in satlist:
aerbcor.add(satfile, arcname=os.path.basename(satfile))
logger.info(f"Add {aerbcor.getnames()}")

# open tar file for writing
with tarfile.open(aerostat, "w") as archive:
for diagfile in diags:
Expand Down