From c3e01ea322c45cfb927f01420faed0e3563059df Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 22 Feb 2024 20:30:31 +0000 Subject: [PATCH 001/189] If using model initial conditions and forcing from the UFS (e.g. Replay mode) with an active LSM, the SCM performs a cold-start initialization for the LSM. That is, UFS replay mode currently support cases generated from UFS Restart files (e.g spun up (warm) LSM parameters) --- scm/etc/scripts/UFS_IC_generator.py | 2 + scm/src/scm_type_defs.F90 | 386 ++++++++-------------------- 2 files changed, 116 insertions(+), 272 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 6fdbd13fd..6004cb39e 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1187,6 +1187,7 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): "facsf": facsf_in, "facwf": facwf_in, "soiltyp": styp_in, + "scolor": 0., "slopetyp": slope_in, "vegtyp": vtyp_in, "vegfrac": vfrac_in, @@ -2450,6 +2451,7 @@ def write_SCM_case_file(state, surface, oro, forcing, case, date, stateREGRID): {"name": "q2m", "type":wp, "dimd": ('t0'), "units": "kg kg-1", "desc": "2-meter specific humidity"}, \ {"name": "vegtyp", "type":wi, "dimd": ('t0'), "units": "none", "desc": "vegetation type (1-12)"}, \ {"name": "soiltyp", "type":wi, "dimd": ('t0'), "units": "none", "desc": "soil type (1-12)"}, \ + {"name": "scolor", "type":wp, "dimd": ('t0'), "units": "none", "desc": "soil color"}, \ {"name": "ffmm", "type":wp, "dimd": ('t0'), "units": "none", "desc": "Monin-Obukhov similarity function for momentum"}, \ {"name": "ffhh", "type":wp, "dimd": ('t0'), "units": "none", "desc": "Monin-Obukhov similarity function for heat"}, \ {"name": "hice", "type":wp, "dimd": ('t0'), "units": "m", "desc": "sea ice thickness"}, \ diff --git a/scm/src/scm_type_defs.F90 b/scm/src/scm_type_defs.F90 index df026e067..550f7d951 100644 --- a/scm/src/scm_type_defs.F90 +++ b/scm/src/scm_type_defs.F90 @@ -1044,27 +1044,6 @@ subroutine physics_set(physics, scm_input, scm_state) logical :: missing_var(100) real, parameter:: min_lake_orog = 200.0_dp - !check whether input has NoahMP or RUC LSM input data - if (scm_state%model_ics .or. scm_state%lsm_ics) then - if (physics%Model%lsm == physics%Model%lsm_noahmp) then - !FV3GFS_io.F90 uses the presence of the snowxy variable in the ICs to indicate presence of NoahMP warm start - call check_missing(scm_input%input_snowxy, missing_var(1)) - if (missing_var(1)) then - physics%Model%lsm_cold_start = .true. - else - physics%Model%lsm_cold_start = .false. - end if - elseif (physics%Model%lsm == physics%Model%lsm_ruc) then - !RUC LSM uses the tslb variable as soil temperature; if it is missing, assume a cold start using Noah LSM ICs - call check_missing(scm_input%input_tslb(:), missing_var(1)) - if (missing_var(1)) then - physics%Model%lsm_cold_start = .true. - else - physics%Model%lsm_cold_start = .false. - end if - end if - end if - !double check under what circumstances these should actually be set from input!!! (these overwrite the initialzation in GFS_typedefs) missing_var = .false. do i = 1, physics%Model%ncols @@ -1074,6 +1053,9 @@ subroutine physics_set(physics, scm_input, scm_state) physics%Sfcprop%landfrac(i) = missing_value physics%Sfcprop%lakefrac(i) = missing_value end if + ! + ! Orographical data (2D) + ! if (scm_state%model_ics) then write(0,'(a)') "Setting internal physics variables from the orographic section of the case input file (scalars)..." call conditionally_set_var(scm_input%input_stddev, physics%Sfcprop%hprime(i,1), "stddev", .true., missing_var(1)) @@ -1096,7 +1078,6 @@ subroutine physics_set(physics, scm_input, scm_state) call conditionally_set_var(scm_input%input_lakefrac, physics%Sfcprop%lakefrac(i), "lakefrac", (physics%Model%lkm == 1), missing_var(18)) call conditionally_set_var(scm_input%input_lakedepth, physics%Sfcprop%lakedepth(i), "lakedepth", (physics%Model%lkm == 1), missing_var(19)) - !write out warning if missing data for non-required variables n = 19 if ( i==1 .and. ANY( missing_var(1:n) ) ) then write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) orography and gravity wave drag parameters. This may lead to crashes or other strange behavior." @@ -1108,55 +1089,58 @@ subroutine physics_set(physics, scm_input, scm_state) missing_var = .false. end if - if (scm_state%model_ics .or. scm_state%lsm_ics) then + ! + ! Surface data (2D) + ! + if (scm_state%model_ics .or. scm_state%lsm_ics) then write(0,'(a)') "Setting internal physics variables from the surface section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_slmsk, physics%Sfcprop%slmsk(i), "slmsk", (.not. physics%Model%frac_grid), missing_var(1)) - call conditionally_set_var(scm_input%input_tsfco, physics%Sfcprop%tsfco(i), "tsfco", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_weasd, physics%Sfcprop%weasd(i), "weasd", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_tg3, physics%Sfcprop%tg3(i), "tg3", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_zorl, physics%Sfcprop%zorl(i), "zorl", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_alvsf, physics%Sfcprop%alvsf(i), "alvsf", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_alnsf, physics%Sfcprop%alnsf(i), "alnsf", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_alvwf, physics%Sfcprop%alvwf(i), "alvwf", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_alnwf, physics%Sfcprop%alnwf(i), "alnwf", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_facsf, physics%Sfcprop%facsf(i), "facsf", .true., missing_var(10)) - call conditionally_set_var(scm_input%input_facwf, physics%Sfcprop%facwf(i), "facwf", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_vegfrac, physics%Sfcprop%vfrac(i), "vegfrac", .true., missing_var(12)) + call conditionally_set_var(scm_input%input_slmsk, physics%Sfcprop%slmsk(i), "slmsk", (.not. physics%Model%frac_grid), missing_var(1)) + call conditionally_set_var(scm_input%input_tsfco, physics%Sfcprop%tsfco(i), "tsfco", .true., missing_var(2)) + call conditionally_set_var(scm_input%input_weasd, physics%Sfcprop%weasd(i), "weasd", .true., missing_var(3)) + call conditionally_set_var(scm_input%input_tg3, physics%Sfcprop%tg3(i), "tg3", .true., missing_var(4)) + call conditionally_set_var(scm_input%input_zorl, physics%Sfcprop%zorl(i), "zorl", .true., missing_var(5)) + call conditionally_set_var(scm_input%input_alvsf, physics%Sfcprop%alvsf(i), "alvsf", .true., missing_var(6)) + call conditionally_set_var(scm_input%input_alnsf, physics%Sfcprop%alnsf(i), "alnsf", .true., missing_var(7)) + call conditionally_set_var(scm_input%input_alvwf, physics%Sfcprop%alvwf(i), "alvwf", .true., missing_var(8)) + call conditionally_set_var(scm_input%input_alnwf, physics%Sfcprop%alnwf(i), "alnwf", .true., missing_var(9)) + call conditionally_set_var(scm_input%input_facsf, physics%Sfcprop%facsf(i), "facsf", .true., missing_var(10)) + call conditionally_set_var(scm_input%input_facwf, physics%Sfcprop%facwf(i), "facwf", .true., missing_var(11)) + call conditionally_set_var(scm_input%input_vegfrac, physics%Sfcprop%vfrac(i), "vegfrac", .true., missing_var(12)) !GJF: is this needed anymore (not in FV3GFS_io)? physics%Interstitial%sigmaf(i) = min(physics%Sfcprop%vfrac(i),0.01) - call conditionally_set_var(scm_input%input_canopy, physics%Sfcprop%canopy(i), "canopy", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_f10m, physics%Sfcprop%f10m(i), "f10m", .false., missing_var(14)) - call conditionally_set_var(scm_input%input_t2m, physics%Sfcprop%t2m(i), "t2m", physics%Model%cplflx, missing_var(15)) - call conditionally_set_var(scm_input%input_q2m, physics%Sfcprop%q2m(i), "q2m", physics%Model%cplflx, missing_var(16)) - call conditionally_set_var(scm_input%input_vegtyp, physics%Sfcprop%vtype(i), "vegtyp", .true., missing_var(17)) - call conditionally_set_var(scm_input%input_soiltyp, physics%Sfcprop%stype(i), "soiltyp", .true., missing_var(18)) - call conditionally_set_var(scm_input%input_uustar, physics%Sfcprop%uustar(i), "uustar", .true., missing_var(19)) - call conditionally_set_var(scm_input%input_ffmm, physics%Sfcprop%ffmm(i), "ffmm", .false., missing_var(20)) - call conditionally_set_var(scm_input%input_ffhh, physics%Sfcprop%ffhh(i), "ffhh", .false., missing_var(21)) - call conditionally_set_var(scm_input%input_hice, physics%Sfcprop%hice(i), "hice", .true., missing_var(22)) - call conditionally_set_var(scm_input%input_fice, physics%Sfcprop%fice(i), "fice", .true., missing_var(23)) - call conditionally_set_var(scm_input%input_tisfc, physics%Sfcprop%tisfc(i), "tisfc", .true., missing_var(24)) - call conditionally_set_var(scm_input%input_tprcp, physics%Sfcprop%tprcp(i), "tprcp", .false., missing_var(25)) - call conditionally_set_var(scm_input%input_srflag, physics%Sfcprop%srflag(i), "srflag", .false., missing_var(26)) - call conditionally_set_var(scm_input%input_snwdph, physics%Sfcprop%snowd(i), "snwdph", .true., missing_var(27)) - call conditionally_set_var(scm_input%input_shdmin, physics%Sfcprop%shdmin(i), "shdmin", .true., missing_var(28)) - call conditionally_set_var(scm_input%input_shdmax, physics%Sfcprop%shdmax(i), "shdmax", .true., missing_var(29)) - call conditionally_set_var(scm_input%input_slopetype, physics%Sfcprop%slope(i), "slopetyp", .true., missing_var(30)) - call conditionally_set_var(scm_input%input_snoalb, physics%Sfcprop%snoalb(i), "snoalb", .true., missing_var(31)) - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr(i), "sncovr", .false., missing_var(32)) - call conditionally_set_var(scm_input%input_snodl, physics%Sfcprop%snodl(i), "snodl", .false., missing_var(33)) - call conditionally_set_var(scm_input%input_weasdl, physics%Sfcprop%weasdl(i), "weasdl", .false., missing_var(34)) - call conditionally_set_var(scm_input%input_tsfc, physics%Sfcprop%tsfc(i), "tsfc", .false., missing_var(35)) - call conditionally_set_var(scm_input%input_tsfcl, physics%Sfcprop%tsfcl(i), "tsfcl", .false., missing_var(36)) - call conditionally_set_var(scm_input%input_zorlw, physics%Sfcprop%zorlw(i), "zorlw", .false., missing_var(37)) - call conditionally_set_var(scm_input%input_zorll, physics%Sfcprop%zorll(i), "zorll", .false., missing_var(38)) - call conditionally_set_var(scm_input%input_zorli, physics%Sfcprop%zorli(i), "zorli", .false., missing_var(39)) + call conditionally_set_var(scm_input%input_canopy, physics%Sfcprop%canopy(i), "canopy", .true., missing_var(13)) + call conditionally_set_var(scm_input%input_f10m, physics%Sfcprop%f10m(i), "f10m", .false., missing_var(14)) + call conditionally_set_var(scm_input%input_t2m, physics%Sfcprop%t2m(i), "t2m", physics%Model%cplflx, missing_var(15)) + call conditionally_set_var(scm_input%input_q2m, physics%Sfcprop%q2m(i), "q2m", physics%Model%cplflx, missing_var(16)) + call conditionally_set_var(scm_input%input_vegtyp, physics%Sfcprop%vtype(i), "vegtyp", .true., missing_var(17)) + call conditionally_set_var(scm_input%input_soiltyp, physics%Sfcprop%stype(i), "soiltyp", .true., missing_var(18)) + call conditionally_set_var(scm_input%input_uustar, physics%Sfcprop%uustar(i), "uustar", .true., missing_var(19)) + call conditionally_set_var(scm_input%input_ffmm, physics%Sfcprop%ffmm(i), "ffmm", .false., missing_var(20)) + call conditionally_set_var(scm_input%input_ffhh, physics%Sfcprop%ffhh(i), "ffhh", .false., missing_var(21)) + call conditionally_set_var(scm_input%input_hice, physics%Sfcprop%hice(i), "hice", .true., missing_var(22)) + call conditionally_set_var(scm_input%input_fice, physics%Sfcprop%fice(i), "fice", .true., missing_var(23)) + call conditionally_set_var(scm_input%input_tisfc, physics%Sfcprop%tisfc(i), "tisfc", .true., missing_var(24)) + call conditionally_set_var(scm_input%input_tprcp, physics%Sfcprop%tprcp(i), "tprcp", .false., missing_var(25)) + call conditionally_set_var(scm_input%input_srflag, physics%Sfcprop%srflag(i), "srflag", .false., missing_var(26)) + call conditionally_set_var(scm_input%input_snwdph, physics%Sfcprop%snowd(i), "snwdph", .true., missing_var(27)) + call conditionally_set_var(scm_input%input_shdmin, physics%Sfcprop%shdmin(i), "shdmin", .true., missing_var(28)) + call conditionally_set_var(scm_input%input_shdmax, physics%Sfcprop%shdmax(i), "shdmax", .true., missing_var(29)) + call conditionally_set_var(scm_input%input_slopetype, physics%Sfcprop%slope(i), "slopetyp", .true., missing_var(30)) + call conditionally_set_var(scm_input%input_snoalb, physics%Sfcprop%snoalb(i), "snoalb", .true., missing_var(31)) + call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr(i), "sncovr", .false., missing_var(32)) + call conditionally_set_var(scm_input%input_snodl, physics%Sfcprop%snodl(i), "snodl", .false., missing_var(33)) + call conditionally_set_var(scm_input%input_weasdl, physics%Sfcprop%weasdl(i), "weasdl", .false., missing_var(34)) + call conditionally_set_var(scm_input%input_tsfc, physics%Sfcprop%tsfc(i), "tsfc", .false., missing_var(35)) + call conditionally_set_var(scm_input%input_tsfcl, physics%Sfcprop%tsfcl(i), "tsfcl", .false., missing_var(36)) + call conditionally_set_var(scm_input%input_zorlw, physics%Sfcprop%zorlw(i), "zorlw", .false., missing_var(37)) + call conditionally_set_var(scm_input%input_zorll, physics%Sfcprop%zorll(i), "zorll", .false., missing_var(38)) + call conditionally_set_var(scm_input%input_zorli, physics%Sfcprop%zorli(i), "zorli", .false., missing_var(39)) call conditionally_set_var(scm_input%input_albdirvis_lnd, physics%Sfcprop%albdirvis_lnd(i), "albdirvis_lnd", .false., missing_var(40)) call conditionally_set_var(scm_input%input_albdirnir_lnd, physics%Sfcprop%albdirnir_lnd(i), "albdirnir_lnd", .false., missing_var(41)) call conditionally_set_var(scm_input%input_albdifvis_lnd, physics%Sfcprop%albdifvis_lnd(i), "albdifvis_lnd", .false., missing_var(42)) call conditionally_set_var(scm_input%input_albdifnir_lnd, physics%Sfcprop%albdifnir_lnd(i), "albdifnir_lnd", .false., missing_var(43)) - call conditionally_set_var(scm_input%input_emis_lnd, physics%Sfcprop%emis_lnd(i), "emis_lnd", .false., missing_var(44)) - if (physics%Model%use_cice_alb .or. physics%Model%lsm == physics%Model%lsm_ruc) then + call conditionally_set_var(scm_input%input_emis_lnd, physics%Sfcprop%emis_lnd(i), "emis_lnd", .false., missing_var(44)) + if (physics%Model%use_cice_alb) then call conditionally_set_var(scm_input%input_albdirvis_ice, physics%Sfcprop%albdirvis_ice(i), "albdirvis_ice", .false., missing_var(45)) call conditionally_set_var(scm_input%input_albdirnir_ice, physics%Sfcprop%albdirnir_ice(i), "albdirnir_ice", .false., missing_var(46)) call conditionally_set_var(scm_input%input_albdifvis_ice, physics%Sfcprop%albdifvis_ice(i), "albdifvis_ice", .false., missing_var(47)) @@ -1206,12 +1190,10 @@ subroutine physics_set(physics, scm_input, scm_state) if (physics%Sfcprop%slmsk(i) > 1.9_dp) physics%Sfcprop%fice(i) = 1.0 !needed to calculate tsfc and zorl below when model_ics == .false. if (physics%Sfcprop%slmsk(i) < 0.1_dp) physics%Sfcprop%oceanfrac(i) = 1.0 end if - - !this overwrites what is in the suite namelist file -- is that desirable? - !if (scm_state%model_ics) then - ! physics%Model%ivegsrc = scm_input%input_vegsrc - !end if - + + ! + ! Derive physics quantities using surface model ICs. + ! if(scm_state%model_ics .or. scm_state%lsm_ics) then if (physics%Sfcprop%stype(i) == 14 .or. physics%Sfcprop%stype(i)+0.5 <= 0) then physics%Sfcprop%landfrac(i) = real_zero @@ -1311,10 +1293,11 @@ subroutine physics_set(physics, scm_input, scm_state) endif end if - !--- NSSTM variables + ! + ! NSSTM variables + ! if (physics%Model%nstf_name(1) > 0) then - if (physics%Model%nstf_name(2) == 1 .or. .not. (scm_state%model_ics .or. scm_state%lsm_ics)) then ! nsst spinup - !--- nsstm tref + if (physics%Model%nstf_name(2) == 1 .or. .not. (scm_state%model_ics .or. scm_state%lsm_ics)) then physics%Sfcprop%tref(i) = physics%Sfcprop%tsfco(i) physics%Sfcprop%z_c(i) = real_zero physics%Sfcprop%c_0(i) = real_zero @@ -1335,183 +1318,50 @@ subroutine physics_set(physics, scm_input, scm_state) physics%Sfcprop%qrain(i) = real_zero elseif (physics%Model%nstf_name(2) == 0) then ! nsst restart write(0,'(a)') "Setting internal physics variables from the NSST section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_tref, physics%Sfcprop%tref(i), "tref", .true., missing_var(1)) - call conditionally_set_var(scm_input%input_z_c, physics%Sfcprop%z_c(i), "z_c", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_c_0, physics%Sfcprop%c_0(i), "c_0", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_c_d, physics%Sfcprop%c_d(i), "c_d", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_w_0, physics%Sfcprop%w_0(i), "w_0", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_w_d, physics%Sfcprop%w_d(i), "w_d", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_xt, physics%Sfcprop%xt(i), "xt", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_xs, physics%Sfcprop%xs(i), "xs", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_xu, physics%Sfcprop%xu(i), "xu", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_xv, physics%Sfcprop%xv(i), "xv", .true., missing_var(10)) - call conditionally_set_var(scm_input%input_xz, physics%Sfcprop%xz(i), "xz", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_zm, physics%Sfcprop%zm(i), "zm", .true., missing_var(12)) - call conditionally_set_var(scm_input%input_xtts, physics%Sfcprop%xtts(i), "xtts", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_xzts, physics%Sfcprop%xzts(i), "xzts", .true., missing_var(14)) - call conditionally_set_var(scm_input%input_d_conv, physics%Sfcprop%d_conv(i), "d_conv", .true., missing_var(15)) - call conditionally_set_var(scm_input%input_ifd, physics%Sfcprop%ifd(i), "ifd", .true., missing_var(16)) + call conditionally_set_var(scm_input%input_tref, physics%Sfcprop%tref(i), "tref", .true., missing_var(1)) + call conditionally_set_var(scm_input%input_z_c, physics%Sfcprop%z_c(i), "z_c", .true., missing_var(2)) + call conditionally_set_var(scm_input%input_c_0, physics%Sfcprop%c_0(i), "c_0", .true., missing_var(3)) + call conditionally_set_var(scm_input%input_c_d, physics%Sfcprop%c_d(i), "c_d", .true., missing_var(4)) + call conditionally_set_var(scm_input%input_w_0, physics%Sfcprop%w_0(i), "w_0", .true., missing_var(5)) + call conditionally_set_var(scm_input%input_w_d, physics%Sfcprop%w_d(i), "w_d", .true., missing_var(6)) + call conditionally_set_var(scm_input%input_xt, physics%Sfcprop%xt(i), "xt", .true., missing_var(7)) + call conditionally_set_var(scm_input%input_xs, physics%Sfcprop%xs(i), "xs", .true., missing_var(8)) + call conditionally_set_var(scm_input%input_xu, physics%Sfcprop%xu(i), "xu", .true., missing_var(9)) + call conditionally_set_var(scm_input%input_xv, physics%Sfcprop%xv(i), "xv", .true., missing_var(10)) + call conditionally_set_var(scm_input%input_xz, physics%Sfcprop%xz(i), "xz", .true., missing_var(11)) + call conditionally_set_var(scm_input%input_zm, physics%Sfcprop%zm(i), "zm", .true., missing_var(12)) + call conditionally_set_var(scm_input%input_xtts, physics%Sfcprop%xtts(i), "xtts", .true., missing_var(13)) + call conditionally_set_var(scm_input%input_xzts, physics%Sfcprop%xzts(i), "xzts", .true., missing_var(14)) + call conditionally_set_var(scm_input%input_d_conv, physics%Sfcprop%d_conv(i), "d_conv", .true., missing_var(15)) + call conditionally_set_var(scm_input%input_ifd, physics%Sfcprop%ifd(i), "ifd", .true., missing_var(16)) call conditionally_set_var(scm_input%input_dt_cool, physics%Sfcprop%dt_cool(i), "dt_cool", .true., missing_var(17)) - call conditionally_set_var(scm_input%input_qrain, physics%Sfcprop%qrain(i), "qrain", .true., missing_var(18)) + call conditionally_set_var(scm_input%input_qrain, physics%Sfcprop%qrain(i), "qrain", .true., missing_var(18)) ! all NNST variables are required when NNST spin-up is off, so no need to write out warning for missing data (the model would have already stopped) missing_var = .false. endif endif - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc .and. .not. physics%Model%lsm_cold_start) then - !--- Extra RUC LSM variables - write(0,'(a)') "Setting internal physics variables from the RUC LSM section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_wetness, physics%Sfcprop%wetness(i), "wetness", .true., missing_var(1)) - call conditionally_set_var(scm_input%input_clw_surf_land, physics%Sfcprop%clw_surf_land(i), "clw_surf_land", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_clw_surf_ice, physics%Sfcprop%clw_surf_ice(i), "clw_surf_ice", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_qwv_surf_land, physics%Sfcprop%qwv_surf_land(i), "qwv_surf_land", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_qwv_surf_ice, physics%Sfcprop%qwv_surf_ice(i), "qwv_surf_ice", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_tsnow_land, physics%Sfcprop%tsnow_land(i), "tsnow_land", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_tsnow_ice, physics%Sfcprop%tsnow_ice(i), "tsnow_ice", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_snowfallac_land, physics%Sfcprop%snowfallac_land(i), "snowfallac_land", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_snowfallac_ice, physics%Sfcprop%snowfallac_ice(i), "snowfallac_ice", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_sncovr_ice, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .false., missing_var(10)) - call conditionally_set_var(scm_input%input_sfalb_lnd, physics%Sfcprop%sfalb_lnd(i), "sfalb_lnd", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_sfalb_lnd_bck, physics%Sfcprop%sfalb_lnd_bck(i), "sfalb_lnd_bck", .true., missing_var(12)) - call conditionally_set_var(scm_input%input_sfalb_ice, physics%Sfcprop%sfalb_ice(i), "sfalb_ice", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_emis_ice, physics%Sfcprop%emis_ice(i), "emis_ice", .true., missing_var(14)) - if (physics%Model%lsm == physics%Model%lsm_ruc .and. physics%Model%rdlai) then - !when rdlai = T, RUC LSM expects the LAI to be read in, hence the required variable attribute below - call conditionally_set_var(scm_input%input_lai, physics%Sfcprop%xlaixy(i), "lai", .true., missing_var(15)) - end if - - !if sncovr_ice is missing, set to the land value - if(missing_var(10)) then - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .true., missing_var(10)) - end if - - !write out warning if missing data for non-required variables - n = 15 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check gmtb_scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - elseif ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc .and. physics%Model%lsm_cold_start) then - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .true., missing_var(1)) - if (physics%Model%rdlai) then - !when rdlai = T, RUC LSM expects the LAI to be read in, hence the required variable attribute below - call conditionally_set_var(scm_input%input_lai, physics%Sfcprop%xlaixy(i), "lai", .true., missing_var(2)) - end if - - !write out warning if missing data for non-required variables - n = 2 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check gmtb_scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - elseif ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_noahmp) then - write(0,'(a)') "Setting internal physics variables from the NoahMP section of the case input file (scalars)..." - !all of these can be missing, since a method exists to "cold start" these variables - call conditionally_set_var(scm_input%input_snowxy, physics%Sfcprop%snowxy(i), "snowxy", .false., missing_var(1)) - call conditionally_set_var(scm_input%input_tvxy, physics%Sfcprop%tvxy(i), "tvxy", .false., missing_var(2)) - call conditionally_set_var(scm_input%input_tgxy, physics%Sfcprop%tgxy(i), "tgxy", .false., missing_var(3)) - call conditionally_set_var(scm_input%input_canicexy, physics%Sfcprop%canicexy(i), "canicexy", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_canliqxy, physics%Sfcprop%canliqxy(i), "canliqxy", .false., missing_var(5)) - call conditionally_set_var(scm_input%input_eahxy, physics%Sfcprop%eahxy(i), "eahxy", .false., missing_var(6)) - call conditionally_set_var(scm_input%input_tahxy, physics%Sfcprop%tahxy(i), "tahxy", .false., missing_var(7)) - call conditionally_set_var(scm_input%input_cmxy, physics%Sfcprop%cmxy(i), "cmxy", .false., missing_var(8)) - call conditionally_set_var(scm_input%input_chxy, physics%Sfcprop%chxy(i), "chxy", .false., missing_var(9)) - call conditionally_set_var(scm_input%input_fwetxy, physics%Sfcprop%fwetxy(i), "fwetxy", .false., missing_var(10)) - call conditionally_set_var(scm_input%input_sneqvoxy, physics%Sfcprop%sneqvoxy(i), "sneqvoxy", .false., missing_var(11)) - call conditionally_set_var(scm_input%input_alboldxy, physics%Sfcprop%alboldxy(i), "alboldxy", .false., missing_var(12)) - call conditionally_set_var(scm_input%input_qsnowxy, physics%Sfcprop%qsnowxy(i), "qsnowxy", .false., missing_var(13)) - call conditionally_set_var(scm_input%input_wslakexy, physics%Sfcprop%wslakexy(i), "wslakexy", .false., missing_var(14)) - call conditionally_set_var(scm_input%input_zwtxy, physics%Sfcprop%zwtxy(i), "zwtxy", .false., missing_var(15)) - call conditionally_set_var(scm_input%input_waxy, physics%Sfcprop%waxy(i), "waxy", .false., missing_var(16)) - call conditionally_set_var(scm_input%input_wtxy, physics%Sfcprop%wtxy(i), "wtxy", .false., missing_var(17)) - call conditionally_set_var(scm_input%input_lfmassxy, physics%Sfcprop%lfmassxy(i), "lfmassxy", .false., missing_var(18)) - call conditionally_set_var(scm_input%input_rtmassxy, physics%Sfcprop%rtmassxy(i), "rtmassxy", .false., missing_var(19)) - call conditionally_set_var(scm_input%input_stmassxy, physics%Sfcprop%stmassxy(i), "stmassxy", .false., missing_var(20)) - call conditionally_set_var(scm_input%input_woodxy, physics%Sfcprop%woodxy(i), "woodxy", .false., missing_var(21)) - call conditionally_set_var(scm_input%input_stblcpxy, physics%Sfcprop%stblcpxy(i), "stblcpxy", .false., missing_var(22)) - call conditionally_set_var(scm_input%input_fastcpxy, physics%Sfcprop%fastcpxy(i), "fastcpxy", .false., missing_var(23)) - call conditionally_set_var(scm_input%input_xsaixy, physics%Sfcprop%xsaixy(i), "xsaixy", .false., missing_var(24)) - call conditionally_set_var(scm_input%input_xlaixy, physics%Sfcprop%xlaixy(i), "xlaixy", .false., missing_var(25)) - call conditionally_set_var(scm_input%input_taussxy, physics%Sfcprop%taussxy(i), "taussxy", .false., missing_var(26)) - call conditionally_set_var(scm_input%input_smcwtdxy, physics%Sfcprop%smcwtdxy(i), "smcwtdxy", .false., missing_var(27)) - call conditionally_set_var(scm_input%input_deeprechxy, physics%Sfcprop%deeprechxy(i), "deeprechxy", .false., missing_var(28)) - call conditionally_set_var(scm_input%input_rechxy, physics%Sfcprop%rechxy(i), "rechxy", .false., missing_var(29)) - - !write out warning if missing data for non-required variables - n = 29 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables for NoahMP LSM. Due to this, a cold-start algorithm to initialize variables will be used." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - end if - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. (physics%Model%lsm == physics%Model%lsm_noah .or. & - physics%Model%lsm == physics%Model%lsm_noahmp .or. physics%Model%lsm_cold_start)) then - + + ! + ! LSM model ICs (3D) + ! + if (scm_state%model_ics .or. scm_state%lsm_ics) then call conditionally_set_var(scm_input%input_stc(:), physics%Sfcprop%stc(i,:), "stc", .true., missing_var(1)) call conditionally_set_var(scm_input%input_smc(:), physics%Sfcprop%smc(i,:), "smc", .true., missing_var(2)) call conditionally_set_var(scm_input%input_slc(:), physics%Sfcprop%slc(i,:), "slc", .true., missing_var(3)) - if (physics%Model%lsm == physics%Model%lsm_noahmp) then - call conditionally_set_var(scm_input%input_snicexy(:), physics%Sfcprop%snicexy(i,:), "snicexy", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_snliqxy(:), physics%Sfcprop%snliqxy(i,:), "sliqexy", .false., missing_var(5)) - call conditionally_set_var(scm_input%input_tsnoxy(:), physics%Sfcprop%tsnoxy(i,:), "tsnoxy", .false., missing_var(6)) - - call conditionally_set_var(scm_input%input_smoiseq(:), physics%Sfcprop%smoiseq(i,:), "smoiseq", .false., missing_var(7)) - - call conditionally_set_var(scm_input%input_zsnsoxy(:), physics%Sfcprop%zsnsoxy(i,:), "zsnzoxy", .false., missing_var(8)) - - !write out warning if missing data for non-required variables - n = 8 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables for NoahMP LSM. Due to this, a cold-start algorithm to initialize variables will be used." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n + n = 3 + if ( i==1 .and. ANY( missing_var(1:n) ) ) then + write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables needed by the LSM. Due to this, a cold-start algorithm to initialize variables will be used." + write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" + do j=1, n if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - endif - - else if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc) then - call conditionally_set_var(scm_input%input_tslb(:), physics%Sfcprop%tslb(i,:), "tslb", .false., missing_var(1)) - call conditionally_set_var(scm_input%input_smois(:), physics%Sfcprop%smois(i,:), "smois", .false., missing_var(2)) - call conditionally_set_var(scm_input%input_sh2o(:), physics%Sfcprop%sh2o(i,:), "sh2o", .false., missing_var(3)) - call conditionally_set_var(scm_input%input_smfr(:), physics%Sfcprop%keepsmfr(i,:), "smfr", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_flfr(:), physics%Sfcprop%flag_frsoil(i,:), "flfr", .false., missing_var(5)) - - !write out warning if missing data for non-required variables - n = 5 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - end if - - if (scm_state%model_ics .or. scm_state%lsm_ics) then - !check for nonmissing values - call conditionally_set_var(scm_input%input_tiice(:), physics%Sfcprop%tiice(i,:), "tiice", .false., missing_var(1)) - if (missing_var(1)) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to the internal sea ice temperature. These will be set from the internal soil temperature variable (stc)." + end do end if - end if + end if + ! + ! Compute surface fields that may/maynot present in model IC files. + ! if (scm_state%model_ics .or. scm_state%lsm_ics) then if (scm_input%input_snodl <= real_zero) then if (physics%Sfcprop%landfrac(i) > real_zero) then @@ -1563,41 +1413,33 @@ subroutine physics_set(physics, scm_input, scm_state) if (scm_input%input_zorlwav <= real_zero) then physics%Sfcprop%zorlwav(i) = physics%Sfcprop%zorlw(i) !--- compute zorlwav from existing variables end if - - if (physics%Model%lsm_cold_start) then - if(physics%Model%frac_grid .and. (scm_state%model_ics .or. scm_state%lsm_ics)) then ! 3-way composite - if( physics%Model%phour < 1.e-7) physics%Sfcprop%tsfco(i) = max(con_tice, physics%Sfcprop%tsfco(i)) - tem1 = real_one - physics%Sfcprop%landfrac(i) - tem = tem1 * physics%Sfcprop%fice(i) ! tem = ice fraction wrt whole cell - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) * physics%Sfcprop%landfrac(i) & - + physics%Sfcprop%zorli(i) * tem & - + physics%Sfcprop%zorlw(i) * (tem1-tem) - - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) * physics%Sfcprop%landfrac(i) & - + physics%Sfcprop%tisfc(i) * tem & - + physics%Sfcprop%tsfco(i) * (tem1-tem) - else - !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlw - ! physics%Sfcprop%tsfcl(i) = physics%Sfcprop%tsfco(i) - ! physics%Sfcprop%zorll(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%zorli(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%zorl(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfco(i) - if (physics%Sfcprop%slmsk(i) == 1) then - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) - else - tem = real_one - physics%Sfcprop%fice(i) - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorli(i) * physics%Sfcprop%fice(i) & - + physics%Sfcprop%zorlw(i) * tem - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tisfc(i) * physics%Sfcprop%fice(i) & - + physics%Sfcprop%tsfco(i) * tem - endif - endif ! if (Model%frac_grid) - endif !if (physics%Model%lsm_cold_start) - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. MAXVAL(scm_input%input_tiice) < real_zero) then + if(physics%Model%frac_grid .and. (scm_state%model_ics .or. scm_state%lsm_ics) ) then ! 3-way composite + if( physics%Model%phour < 1.e-7) physics%Sfcprop%tsfco(i) = max(con_tice, physics%Sfcprop%tsfco(i)) + tem1 = real_one - physics%Sfcprop%landfrac(i) + tem = tem1 * physics%Sfcprop%fice(i) ! tem = ice fraction wrt whole cell + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) * physics%Sfcprop%landfrac(i) & + + physics%Sfcprop%zorli(i) * tem & + + physics%Sfcprop%zorlw(i) * (tem1-tem) + + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) * physics%Sfcprop%landfrac(i) & + + physics%Sfcprop%tisfc(i) * tem & + + physics%Sfcprop%tsfco(i) * (tem1-tem) + else + if (physics%Sfcprop%slmsk(i) == 1) then + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) + else + tem = real_one - physics%Sfcprop%fice(i) + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorli(i) * physics%Sfcprop%fice(i) & + + physics%Sfcprop%zorlw(i) * tem + + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tisfc(i) * physics%Sfcprop%fice(i) & + + physics%Sfcprop%tsfco(i) * tem + endif + endif ! if (Model%frac_grid) + + if (scm_state%model_ics .and. MAXVAL(scm_input%input_tiice) < real_zero) then physics%Sfcprop%tiice(i,1) = physics%Sfcprop%stc(i,1) !--- initialize internal ice temp from soil temp at layer 1 physics%Sfcprop%tiice(i,2) = physics%Sfcprop%stc(i,2) !--- initialize internal ice temp from soil temp at layer 2 end if From 53548eaedbc49d75468edeed0eef5553ad6d9d49 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 28 Feb 2024 22:01:05 +0000 Subject: [PATCH 002/189] Some changes for Man to test --- scm/etc/scripts/UFS_IC_generator.py | 14 +++++++++----- scm/src/scm_input.F90 | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 6004cb39e..2e3df2b68 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1066,10 +1066,14 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): tprcp_in = read_NetCDF_surface_var(nc_file, 'tprcp', i, j, old_chgres, 0) srflag_in = read_NetCDF_surface_var(nc_file, 'srflag', i, j, old_chgres, 0) sncovr_in = read_NetCDF_surface_var(nc_file, 'sncovr', i, j, old_chgres, 0) - tsfcl_in = read_NetCDF_surface_var(nc_file, 'tsfcl', i, j, old_chgres, 0) - zorll_in = read_NetCDF_surface_var(nc_file, 'zorll', i, j, old_chgres, 0) - zorli_in = read_NetCDF_surface_var(nc_file, 'zorli', i, j, old_chgres, 0) - + tsfcl_in = read_NetCDF_surface_var(nc_file, 'tsea', i, j, old_chgres, 0) + zorll_in = read_NetCDF_surface_var(nc_file, 'zorl', i, j, old_chgres, 0) + zorli_in = read_NetCDF_surface_var(nc_file, 'zorl', i, j, old_chgres, 0) + if (snwdph_in > 0): + sncovr_in = 1.0 + else: + sncovr_in = 0.0 + # present when cplwav = T zorlw_in = read_NetCDF_surface_var(nc_file, 'zorlw', i, j, old_chgres, 0) @@ -2393,7 +2397,7 @@ def write_SCM_case_file(state, surface, oro, forcing, case, date, stateREGRID): {"name": "mrsos_forc", "type":wp, "dimd": ('time' ), "units": "kg m-2", "desc": "forcing_mass_content_of_water_in_soil_layer"}] # - var_oro = [{"name": "area", "type":wp, "dimd": ('t0'), "units": "m 2-1", "desc": "grid_cell_area"},\ + var_oro = [{"name": "area", "type":wp, "dimd": ('t0'), "units": "m2", "desc": "grid_cell_area"},\ {"name": "stddev", "type":wp, "dimd": ('t0'), "units": "m", "desc": "standard deviation of subgrid orography"}, \ {"name": "convexity", "type":wp, "dimd": ('t0'), "units": "none", "desc": "convexity of subgrid orography"}, \ {"name": "oa1", "type":wp, "dimd": ('t0'), "units": "none", "desc": "assymetry of subgrid orography 1"}, \ diff --git a/scm/src/scm_input.F90 b/scm/src/scm_input.F90 index a39404366..3269c16ac 100644 --- a/scm/src/scm_input.F90 +++ b/scm/src/scm_input.F90 @@ -1969,6 +1969,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) if (trim(input_surfaceForcingLSM) == "lsm") then scm_input%input_ozone = input_ozone(:,active_init_time) scm_input%input_area = input_area(active_init_time) + scm_state%area = input_area(active_init_time) scm_input%input_stddev = input_stddev(active_init_time) scm_input%input_convexity= input_convexity(active_init_time) From f22b07e1fe16449ae341750e948b91982a606744 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 28 Feb 2024 22:42:50 +0000 Subject: [PATCH 003/189] Replay with RUC and NoahMP LSM working --- scm/etc/scripts/UFS_IC_generator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 2e3df2b68..af3f45d29 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1175,6 +1175,12 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): # fractional grid tiice_in = read_NetCDF_surface_var(nc_file, 'tiice', i, j, old_chgres, missing_variable_ice_layers) + # soil color + if (slmsk_in == 1): + scolor_in = 4 + else: + scolor_in = 1 + # nc_file.close() @@ -1191,7 +1197,7 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): "facsf": facsf_in, "facwf": facwf_in, "soiltyp": styp_in, - "scolor": 0., + "scolor": scolor_in, "slopetyp": slope_in, "vegtyp": vtyp_in, "vegfrac": vfrac_in, From 8e21d36301c6c9b583c397d6fabb8ffe960e303c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 15:17:14 +0000 Subject: [PATCH 004/189] Small cahnge from physics needed for replay --- .gitmodules | 4 ++-- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index dc0798c32..b1421ef8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics - branch = main + url = https://github.com/dustinswales/ccpp-physics + branch = replay_fixLSMinit [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 23f120a54..832dcced3 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 23f120a54ac1de039a496f249b07532f07cfb38b +Subproject commit 832dcced3692bd9c0d3526ebb622f79a02413d1a From 3f06191655cfb0785f49a1e05c7154d51cd19087 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 15:19:28 +0000 Subject: [PATCH 005/189] Synced physics --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 832dcced3..aabe63844 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 832dcced3692bd9c0d3526ebb622f79a02413d1a +Subproject commit aabe63844ae4a81cc20d1937d737c078380a0392 From 8107c705a0e2f32af773e6b82a9ec8f1dd2eb3e0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 16:54:37 +0000 Subject: [PATCH 006/189] Foundatio for Nvidia CI script --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml new file mode 100644 index 000000000..c976020b6 --- /dev/null +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -0,0 +1,182 @@ +name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers + +on: [push,pull_request,workflow_dispatch] + +jobs: + build_scm: + + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + strategy: + matrix: + fortran-compiler: [nvfortran] + build-type: [Release, Debug] + py-version: [3.7.13, 3.9.12] + + # Environmental variables + env: + NFHOME: /home/runner/netcdf-fortran + NFVERSION: v4.5.3 + bacio_ROOT: /home/runner/bacio + sp_ROOT: /home/runner/NCEPLIBS-sp + w3emc_ROOT: /home/runner/myw3emc + SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + intel_ROOT: /home/runner/intel + suites: SCM_RAP + + # Workflow steps + steps: + ####################################################################################### + # Initial + ####################################################################################### + - name: Checkout SCM code (into /home/runner/work/ccpp-scm/) + uses: actions/checkout@v3 + + - name: Initialize submodules + run: git submodule update --init --recursive + + ####################################################################################### + # Python setup + ####################################################################################### + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.py-version}} + + - name: Add conda to system path + run: | + echo $CONDA/bin >> $GITHUB_PATH + + - name: Install NetCDF Python libraries + run: | + conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml + + - name: Update system packages + run: sudo apt-get update + + ####################################################################################### + # Install Nvidia. + ####################################################################################### + - name: cache-nvidia-compilers + id: cache-nvidia-compilers + if: contains(matrix.fortran-compiler, 'nvfortran') + uses: actions/cache@v3 + with: + path: /opt/nvidia/hpc_sdk/ + key: nvhpc-${{ runner.os }}-2024-24.1 + + - name: Nvidia setup compilers + if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' + env: + NVCOMPILERS: /opt/nvidia/hpc_sdk + run: | + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + + ####################################################################################### + # Install FORTRAN dependencies + ####################################################################################### + - name: Environment for nvidia compiler. + if: contains(matrix.fortran-compiler, 'nvfortran') + run: | + echo "CC=nvc" >> $GITHUB_ENV + echo "FC=nvfortran" >> $GITHUB_ENV + echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV + echo "NFVERSION=v4.4.4" >> $GITHUB_ENV + + - name: Cache bacio library v2.4.1 + id: cache-bacio-fortran + uses: actions/cache@v3 + with: + path: /home/runner/bacio + key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key + + - name: Install bacio library v2.4.1 + if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio + cd bacio && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${bacio_ROOT} ../ + make -j2 + make install + echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV + + - name: Cache SP-library v2.3.3 + id: cache-sp-fortran + uses: actions/cache@v3 + with: + path: /home/runner/NCEPLIBS-sp + key: cache-sp-fortran-${{matrix.fortran-compiler}}-key + + - name: Install SP-library v2.3.3 + if: steps.cache-sp-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp + cd NCEPLIBS-sp && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${sp_ROOT} ../ + make -j2 + make install + echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV + + - name: Cache w3emc library v2.9.2 + id: cache-w3emc-fortran + uses: actions/cache@v3 + with: + path: /home/runner/myw3emc + key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key + + - name: Install w3emc library v2.9.2 + if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc + cd NCEPLIBS-w3emc && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${w3emc_ROOT} ../ + make -j2 + make install + echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + + - name: Install NetCDF C library + run: | + sudo apt-get update + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + + ####################################################################################### + # Build SCM. + ####################################################################################### + + - name: Configure build with CMake (Release) + if: contains(matrix.build-type, 'Release') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} ../src + + - name: Configure build with CMake (Debug) + if: contains(matrix.build-type, 'Debug') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + + - name: Build SCM + run: | + cd ${SCM_ROOT}/scm/bin + make -j4 From 271710bf7692d4f558bd17c0d0a431f081a19ae7 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:05:47 +0000 Subject: [PATCH 007/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c976020b6..e9445216c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,19 +10,19 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] - py-version: [3.7.13, 3.9.12] + build-type: [Release]#, Debug] + py-version: [3.7.13]#, 3.9.12] # Environmental variables env: - NFHOME: /home/runner/netcdf-fortran - NFVERSION: v4.5.3 - bacio_ROOT: /home/runner/bacio - sp_ROOT: /home/runner/NCEPLIBS-sp - w3emc_ROOT: /home/runner/myw3emc - SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - intel_ROOT: /home/runner/intel - suites: SCM_RAP + NFHOME: /home/runner/netcdf-fortran + NFVERSION: v4.5.3 + bacio_ROOT: /home/runner/bacio + sp_ROOT: /home/runner/NCEPLIBS-sp + w3emc_ROOT: /home/runner/myw3emc + SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + nvidia_ROOT: /home/runner/intel + suites: SCM_RAP # Workflow steps steps: @@ -62,7 +62,7 @@ jobs: if: contains(matrix.fortran-compiler, 'nvfortran') uses: actions/cache@v3 with: - path: /opt/nvidia/hpc_sdk/ + path: /home/runner/hpc_sdk/ key: nvhpc-${{ runner.os }}-2024-24.1 - name: Nvidia setup compilers @@ -70,9 +70,13 @@ jobs: env: NVCOMPILERS: /opt/nvidia/hpc_sdk run: | - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg + echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list + sudo apt-get update -y + sudo apt-get install -y nvhpc-24-1 +# wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz +# tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz +# nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ####################################################################################### # Install FORTRAN dependencies From 2321196bbe439b8780ff098ba96839cdbe95e7ae Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:14:32 +0000 Subject: [PATCH 008/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e9445216c..a5adf5f67 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -74,9 +74,6 @@ jobs: echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y sudo apt-get install -y nvhpc-24-1 -# wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz -# tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz -# nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ####################################################################################### # Install FORTRAN dependencies @@ -139,48 +136,3 @@ jobs: make -j2 make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - - name: Install NetCDF C library - run: | - sudo apt-get update - sudo apt-get install libnetcdf-dev - - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - - - name: Install NetCDF Fortran library - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib - - ####################################################################################### - # Build SCM. - ####################################################################################### - - - name: Configure build with CMake (Release) - if: contains(matrix.build-type, 'Release') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src - - - name: Configure build with CMake (Debug) - if: contains(matrix.build-type, 'Debug') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src - - - name: Build SCM - run: | - cd ${SCM_ROOT}/scm/bin - make -j4 From 0bfbe997b333be71bd4386e07d0e65caa970ffb3 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:28:41 +0000 Subject: [PATCH 009/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a5adf5f67..ab9f76274 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,7 +21,7 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - nvidia_ROOT: /home/runner/intel + nvidia_ROOT: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -68,17 +68,19 @@ jobs: - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: - NVCOMPILERS: /opt/nvidia/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk run: | curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -y nvhpc-24-1 + sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk/ -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. + env: + NVCOMPILERS: /home/runner/hpc_sdk if: contains(matrix.fortran-compiler, 'nvfortran') run: | echo "CC=nvc" >> $GITHUB_ENV From c6511633dc6946b35409a668a0976011eff392a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:41:23 +0000 Subject: [PATCH 010/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ab9f76274..5efbb425a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,7 +21,9 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - nvidia_ROOT: /home/runner/nvidia + NVHPC_SILENT: true + NVHPC_INSTALL_TYPE: single + NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -73,7 +75,7 @@ jobs: curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk/ -y nvhpc-24-1 + sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies From 915bd81d535fb5b30cd5782f99560b0bc2c98ed4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:48:04 +0000 Subject: [PATCH 011/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5efbb425a..260f36c89 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -75,7 +75,7 @@ jobs: curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk -y nvhpc-24-1 + sudo apt-get install -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies From 6aba5e8f9dc6680055e020d56b7aa975c9f6c34c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:02:26 +0000 Subject: [PATCH 012/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 260f36c89..45d06cfac 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -28,6 +28,12 @@ jobs: # Workflow steps steps: + + - name: Clear caches + uses: easimon/wipe-cache@main + with: + dry-run: 'true' + ####################################################################################### # Initial ####################################################################################### From 4ec0f3dc9982da948c5c9bf066d6700ff99df79a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:13:27 +0000 Subject: [PATCH 013/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 45d06cfac..64e5b017e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -28,12 +28,10 @@ jobs: # Workflow steps steps: - - - name: Clear caches - uses: easimon/wipe-cache@main - with: - dry-run: 'true' - +# - name: Clear caches +# uses: easimon/wipe-cache@main +# with: +# dry-run: 'true' ####################################################################################### # Initial ####################################################################################### @@ -71,7 +69,7 @@ jobs: uses: actions/cache@v3 with: path: /home/runner/hpc_sdk/ - key: nvhpc-${{ runner.os }}-2024-24.1 + key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' @@ -96,15 +94,15 @@ jobs: echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - - name: Cache bacio library v2.4.1 - id: cache-bacio-fortran - uses: actions/cache@v3 - with: - path: /home/runner/bacio - key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache bacio library v2.4.1 +# id: cache-bacio-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/bacio +# key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key - name: Install bacio library v2.4.1 - if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' +# if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio cd bacio && mkdir build && cd build @@ -113,15 +111,15 @@ jobs: make install echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV - - name: Cache SP-library v2.3.3 - id: cache-sp-fortran - uses: actions/cache@v3 - with: - path: /home/runner/NCEPLIBS-sp - key: cache-sp-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache SP-library v2.3.3 +# id: cache-sp-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/NCEPLIBS-sp +# key: cache-sp-fortran-${{matrix.fortran-compiler}}-key - name: Install SP-library v2.3.3 - if: steps.cache-sp-fortran.outputs.cache-hit != 'true' +# if: steps.cache-sp-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp cd NCEPLIBS-sp && mkdir build && cd build @@ -130,15 +128,15 @@ jobs: make install echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV - - name: Cache w3emc library v2.9.2 - id: cache-w3emc-fortran - uses: actions/cache@v3 - with: - path: /home/runner/myw3emc - key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache w3emc library v2.9.2 +# id: cache-w3emc-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/myw3emc +# key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key - name: Install w3emc library v2.9.2 - if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' +# if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc cd NCEPLIBS-w3emc && mkdir build && cd build From 3474475fb799dac0e4a78150c6fe4131411db38e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:22:19 +0000 Subject: [PATCH 014/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 64e5b017e..b1db8370c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -85,9 +85,6 @@ jobs: # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. - env: - NVCOMPILERS: /home/runner/hpc_sdk - if: contains(matrix.fortran-compiler, 'nvfortran') run: | echo "CC=nvc" >> $GITHUB_ENV echo "FC=nvfortran" >> $GITHUB_ENV From 2065f86d5bb15fbb91a814c47331239628c2d6a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:34:13 +0000 Subject: [PATCH 015/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b1db8370c..a8ff85974 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -80,6 +80,10 @@ jobs: echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y sudo apt-get install -y nvhpc-24-1 + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From abf075597aa4a2603f4d038bb5e7f3ae226f8c3a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:21:28 +0000 Subject: [PATCH 016/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a8ff85974..608fb3df7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -76,10 +76,14 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | - curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg - echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list - sudo apt-get update -y - sudo apt-get install -y nvhpc-24-1 + mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install +# curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg +# echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list +# sudo apt-get update -y +# sudo apt-get install -y nvhpc-24-1 echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 53260ae124fc0b397673bb9993a7f55ac25d031d Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:24:16 +0000 Subject: [PATCH 017/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 608fb3df7..29e345f27 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -63,31 +63,31 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: cache-nvidia-compilers - id: cache-nvidia-compilers - if: contains(matrix.fortran-compiler, 'nvfortran') - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk/ - key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key - +# - name: cache-nvidia-compilers +# id: cache-nvidia-compilers +# if: contains(matrix.fortran-compiler, 'nvfortran') +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk/ +# key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key +# - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk run: | - mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install # curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg # echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list # sudo apt-get update -y # sudo apt-get install -y nvhpc-24-1 - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 09bad92b9e6abc18ce976843e992a9b225f69c70 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:26:28 +0000 Subject: [PATCH 018/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 29e345f27..4de98f44d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -63,14 +63,6 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: cache-nvidia-compilers -# id: cache-nvidia-compilers -# if: contains(matrix.fortran-compiler, 'nvfortran') -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk/ -# key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key -# - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: @@ -80,10 +72,6 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install -# curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg -# echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list -# sudo apt-get update -y -# sudo apt-get install -y nvhpc-24-1 echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 95fac483665c381a8c66352108ef37cbd88937bc Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:15:58 +0000 Subject: [PATCH 019/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4de98f44d..189e35f0c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,8 +21,8 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - NVHPC_SILENT: true - NVHPC_INSTALL_TYPE: single +# NVHPC_SILENT: true +# NVHPC_INSTALL_TYPE: single NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP From 79a3e845e93c724f911eb919fd7459f7e3fab537 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:23:22 +0000 Subject: [PATCH 020/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 189e35f0c..6baa3e840 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,7 +23,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm # NVHPC_SILENT: true # NVHPC_INSTALL_TYPE: single - NVHPC_INSTALL_DIR: /home/runner/nvidia +# NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -68,6 +68,9 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | + echo "NVHPC_SILENT=true" >> $GITHUB_ENV + echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 4e242d63c1b1b43c235e543b4a6626f66e1b5c22 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:24:16 +0000 Subject: [PATCH 021/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6baa3e840..c371584eb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -69,8 +69,8 @@ jobs: NVCOMPILERS: /home/runner/hpc_sdk run: | echo "NVHPC_SILENT=true" >> $GITHUB_ENV - echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 0621092d50dfacc8233fec167ee5895bc5dc4631 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:48:01 +0000 Subject: [PATCH 022/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c371584eb..6e3deade7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -52,13 +52,16 @@ jobs: - name: Add conda to system path run: | echo $CONDA/bin >> $GITHUB_PATH + df -h - name: Install NetCDF Python libraries run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - + df -h - name: Update system packages - run: sudo apt-get update + run: | + sudo apt-get update + df -h ####################################################################################### # Install Nvidia. @@ -68,6 +71,7 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | + df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV From 091348804d6c70a5758c2793c65f51cf41eb3d43 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:59:35 +0000 Subject: [PATCH 023/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6e3deade7..d26f5bce6 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,8 +3,27 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: - build_scm: + free-disk-space: + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + build_scm: # The type of runner that the job will run on runs-on: ubuntu-22.04 strategy: From e3ef9301c882dcacfbde9543d8f2eea118056f96 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:04:58 +0000 Subject: [PATCH 024/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d26f5bce6..a00f0dca8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,25 +3,25 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: - free-disk-space: - runs-on: ubuntu-latest - steps: - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true +# free-disk-space: +# runs-on: ubuntu-latest +# steps: +# +# - name: Free Disk Space (Ubuntu) +# uses: jlumbroso/free-disk-space@main +# with: +# # this might remove tools that are actually needed, +# # if set to "true" but frees about 6 GB +# tool-cache: false +# +# # all of these default to true, but feel free to set to +# # "false" if necessary for your workflow +# android: true +# dotnet: true +# haskell: true +# large-packages: true +# docker-images: true +# swap-storage: true build_scm: # The type of runner that the job will run on @@ -40,17 +40,38 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm -# NVHPC_SILENT: true -# NVHPC_INSTALL_TYPE: single -# NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps steps: -# - name: Clear caches -# uses: easimon/wipe-cache@main -# with: -# dry-run: 'true' + + ####################################################################################### + # Cleanup space + ####################################################################################### + - name: Check space (pre) + run: | + df -h + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Check space (post) + run: | + df -h + ####################################################################################### # Initial ####################################################################################### From 27c6225db1e3e9042e52c14f1fc246402d6e2497 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:08:33 +0000 Subject: [PATCH 025/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a00f0dca8..b5b77eda9 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -70,7 +70,7 @@ jobs: - name: Check space (post) run: | - df -h + df -h ####################################################################################### # Initial From 9eeb2cc43f2fd188836934c161451f4cb696bc31 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:42:29 +0000 Subject: [PATCH 026/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b5b77eda9..5e121aa43 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,6 +110,9 @@ jobs: if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk + NVHPC_SILENT: true + NVHPC_INSTALL_DIR: /home/runner/hpc_sdk + NVHPC_INSTALL_TYPE: single run: | df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV From ab09969af55d9d7574f3051a172e31aa6be1d092 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 23:13:55 +0000 Subject: [PATCH 027/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5e121aa43..36dadf9e7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -122,10 +122,10 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran + #echo "The nvfortran installed is:" + #nvfortran --version + #echo "The path to nvfortran is:" + #command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 29d6e4e3de3c4052c1b247100d2933e92e44ffb8 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:30:55 +0000 Subject: [PATCH 028/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 36dadf9e7..101d2a652 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,25 +3,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: -# free-disk-space: -# runs-on: ubuntu-latest -# steps: -# -# - name: Free Disk Space (Ubuntu) -# uses: jlumbroso/free-disk-space@main -# with: -# # this might remove tools that are actually needed, -# # if set to "true" but frees about 6 GB -# tool-cache: false -# -# # all of these default to true, but feel free to set to -# # "false" if necessary for your workflow -# android: true -# dotnet: true -# haskell: true -# large-packages: true -# docker-images: true -# swap-storage: true build_scm: # The type of runner that the job will run on @@ -106,8 +87,15 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### +# - name: Cache Nvidia HPC +# id: cache-nvidia +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk +# key: cache-nvidia-key + - name: Nvidia setup compilers - if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' +# if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true From 17f0e73182da65af1910adb33baa840ae9619197 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:45:19 +0000 Subject: [PATCH 029/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 101d2a652..20bfed11a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,6 +110,7 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + ls /home/runner/hpc_sdk #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From 4ca35b27c1177c4880b97d016e75839bdccbf584 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:53:40 +0000 Subject: [PATCH 030/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 20bfed11a..000f02b15 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,6 +111,16 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk + echo "NVARCH=`uname -s`_`uname -m`; export NVARCH" >> $GITHUB_ENV + echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV + echo "MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV + +$ NVARCH=`uname -s`_`uname -m`; export NVARCH +$ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS +$ MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH +$ PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH + #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From 69d0945d02a64bd070207e3ec29a60874145d622 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 03:04:58 +0000 Subject: [PATCH 031/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 000f02b15..cfae973fd 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,10 +111,11 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk - echo "NVARCH=`uname -s`_`uname -m`; export NVARCH" >> $GITHUB_ENV - echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV - echo "MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV + NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS + PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH + LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH +# echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV +# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV $ NVARCH=`uname -s`_`uname -m`; export NVARCH $ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS From 66c067fb52610dc8a5ecbe51566815b095f3b051 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 03:06:36 +0000 Subject: [PATCH 032/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cfae973fd..b75b379f3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -117,10 +117,10 @@ jobs: # echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV -$ NVARCH=`uname -s`_`uname -m`; export NVARCH -$ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS -$ MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH -$ PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH +# NVARCH=`uname -s`_`uname -m`; export NVARCH +# NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS +# MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH +# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH #echo "The nvfortran installed is:" #nvfortran --version From 584ae012bddb06cdcad306052e2123a795ae7116 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 16:54:26 +0000 Subject: [PATCH 033/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b75b379f3..ba80d82bd 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,11 +111,11 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk - NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS - PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH - LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH -# echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV -# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV +# NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS +# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH +# LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV # NVARCH=`uname -s`_`uname -m`; export NVARCH # NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS @@ -136,6 +136,7 @@ jobs: echo "FC=nvfortran" >> $GITHUB_ENV echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV echo "NFVERSION=v4.4.4" >> $GITHUB_ENV + ls /home/runner/hpc_sdk # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 98eaa8a54ed691cd5acdec4f1d7aad7fefac9fea Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 16:56:52 +0000 Subject: [PATCH 034/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ba80d82bd..648330f09 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -95,7 +95,7 @@ jobs: # key: cache-nvidia-key - name: Nvidia setup compilers -# if: steps.cache-nvidia.outputs.cache-hit != 'true' + if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true @@ -111,17 +111,8 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk -# NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS -# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH -# LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - -# NVARCH=`uname -s`_`uname -m`; export NVARCH -# NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS -# MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH -# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH - #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From e95f7045e4fc7016cf834b39c3d9590bb5e3e258 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:29:38 +0000 Subject: [PATCH 035/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 648330f09..3aef9c8da 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -102,7 +102,6 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV @@ -110,13 +109,14 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - ls /home/runner/hpc_sdk - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - #echo "The nvfortran installed is:" - #nvfortran --version - #echo "The path to nvfortran is:" - #command -v nvfortran + echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 62a4ca6fda4fbcd9588a9d8edbf467b62ba4ad4c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:46:53 +0000 Subject: [PATCH 036/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3aef9c8da..ea274cf9b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,22 +97,27 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - NVCOMPILERS: /home/runner/hpc_sdk +# NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - echo "NVHPC_SILENT=true" >> $GITHUB_ENV + echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "SWALES here1" + echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV + echo "SWALES here2" + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "SWALES here3" echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "SWALES here4" + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 78365adb5121a57c76c6eb2f88962d8d40e7549c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:48:28 +0000 Subject: [PATCH 037/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ea274cf9b..f981ee7fb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,7 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: -# NVCOMPILERS: /home/runner/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single From 7cf9e400908597ff6d9dd016e9cad54c510a61b0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:50:59 +0000 Subject: [PATCH 038/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f981ee7fb..38e30ba2f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,7 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - NVCOMPILERS: /home/runner/hpc_sdk + #NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single @@ -111,13 +111,13 @@ jobs: nvhpc_2024_241_Linux_x86_64_cuda_12.3/install echo "SWALES here1" echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "SWALES here2" + echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "SWALES here3" + echo "SWALES here3" echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV - echo "SWALES here4" + echo "SWALES here4" echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - echo "SWALES here5" + echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From eb53bde61cd802762835ffdaa3e1f98c3557fb07 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:06:15 +0000 Subject: [PATCH 039/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 38e30ba2f..f1451131e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,7 +114,9 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" - echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + echo "SWALES here3.5" + echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" From 55921f7b16b8d9552a0d42c1013e00331ef592b1 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:17:30 +0000 Subject: [PATCH 040/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f1451131e..8bafac11e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -115,10 +115,11 @@ jobs: echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - echo "SWALES here3.5" - echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV +# echo "SWALES here3.5" +# echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + export PATH=${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin:${PATH} +# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From cf076f905b00befb83b423cb1c1f892c5d2537c0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:22:53 +0000 Subject: [PATCH 041/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8bafac11e..8eed37a99 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -115,10 +115,10 @@ jobs: echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man -# echo "SWALES here3.5" +# echo "SWALES here3.5" # echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - export PATH=${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin:${PATH} + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 1ef18b3f1c15025621d29aeab3e5b8923e0c8bb4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:24:10 +0000 Subject: [PATCH 042/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8eed37a99..978d0dc46 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -118,7 +118,7 @@ jobs: # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 20e4948df7e66aec51d1762797ec8265b9e7e336 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:26:54 +0000 Subject: [PATCH 043/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 978d0dc46..e352db3c1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,12 +114,12 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man +# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" -# echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV + echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" # export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin -# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 629908ec0eeb4260c14cfc82f74db0b946686199 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:27:35 +0000 Subject: [PATCH 044/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e352db3c1..7bca21b51 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,12 +114,12 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" -# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" - echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV +# echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" -# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 31cae533caa0b60fbd8aa9b7dfecf1f25755eaa0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:28:38 +0000 Subject: [PATCH 045/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7bca21b51..5f3f2861b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -113,12 +113,12 @@ jobs: echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "SWALES here3" - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man +# echo "SWALES here3" +# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV - echo "SWALES here4" - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# echo "SWALES here4" +# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 880165dec370d8ca200f4a20ff544bb2ef1f0b38 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:29:59 +0000 Subject: [PATCH 046/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5f3f2861b..fd1f42d0e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,8 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - #NVCOMPILERS: /home/runner/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk + NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single @@ -109,16 +110,16 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "SWALES here1" - echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "SWALES here2" - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV +# echo "SWALES here1" +# echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV +# echo "SWALES here2" +# echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV # echo "SWALES here3" # export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV # echo "SWALES here4" -# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 3fc5f81dd0db6f11d45c3868ee389f56608ea64c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:31:44 +0000 Subject: [PATCH 047/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index fd1f42d0e..bd97c2761 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,17 +110,8 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install -# echo "SWALES here1" -# echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV -# echo "SWALES here2" -# echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV -# echo "SWALES here3" -# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man -# echo "SWALES here3.5" -# echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV -# echo "SWALES here4" + echo "SWALES here4" export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin -# echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 2194f2e71cee00ed402a2f5060d1d63d125afa49 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:42:03 +0000 Subject: [PATCH 048/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bd97c2761..edb7863c5 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -87,12 +87,12 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: Cache Nvidia HPC -# id: cache-nvidia -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk -# key: cache-nvidia-key + - name: Cache Nvidia HPC + id: cache-nvidia + uses: actions/cache@v3 + with: + path: /home/runner/hpc_sdk + key: cache-nvidia-key - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' @@ -103,9 +103,9 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - echo "NVHPC_SILENT=true" >> $GITHUB_ENV - echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + #echo "NVHPC_SILENT=true" >> $GITHUB_ENV + #echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + #echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 3bd379650a803ea7313017ab2c8ff1aea928d936 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:52:44 +0000 Subject: [PATCH 049/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index edb7863c5..3f7c7b1ed 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -103,31 +103,27 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - #echo "NVHPC_SILENT=true" >> $GITHUB_ENV - #echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - #echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "SWALES here4" export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" command -v nvfortran + echo "Removing tarball" + rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. run: | - echo "CC=nvc" >> $GITHUB_ENV - echo "FC=nvfortran" >> $GITHUB_ENV - echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV - echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - ls /home/runner/hpc_sdk + export CC=nvc + export FC=nvfortran + export F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk + export NFVERSION=v4.4.4 # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 7d9c4af70bdca79da9159b00ee52e4f33dc1af50 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:11:24 +0000 Subject: [PATCH 050/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3f7c7b1ed..20a4b17e4 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -22,6 +22,9 @@ jobs: w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP + F90FLAGS: -O3 + CC: nvc + FC: nvfortran # Workflow steps steps: @@ -122,8 +125,6 @@ jobs: run: | export CC=nvc export FC=nvfortran - export F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk - export NFVERSION=v4.4.4 # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 19081c6e04c8b52b593e4633d59ab985e7787cc2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:25:26 +0000 Subject: [PATCH 051/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 20a4b17e4..a4af6c635 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,8 +23,8 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP F90FLAGS: -O3 - CC: nvc - FC: nvfortran + #CC: nvc + #FC: nvfortran # Workflow steps steps: @@ -125,6 +125,8 @@ jobs: run: | export CC=nvc export FC=nvfortran + export CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc + export CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 45b76c4ec2308148693919d442e8db80fb924c6e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:37:34 +0000 Subject: [PATCH 052/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a4af6c635..b355a92a0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,8 +23,8 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP F90FLAGS: -O3 - #CC: nvc - #FC: nvfortran + CC: nvc + FC: nvfortran # Workflow steps steps: From 0d8c9e47e477c52795a008235d01a56dd9e932db Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:46:18 +0000 Subject: [PATCH 053/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b355a92a0..247610c33 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,6 +25,8 @@ jobs: F90FLAGS: -O3 CC: nvc FC: nvfortran + CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc + CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # Workflow steps steps: From 3365312b29ad3120d5d1c4c46c7c5a5b579f78c9 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:56:53 +0000 Subject: [PATCH 054/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 247610c33..a0f857c21 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -125,10 +125,10 @@ jobs: ####################################################################################### - name: Environment for nvidia compiler. run: | - export CC=nvc - export FC=nvfortran - export CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc - export CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + echo "CC=nvc" >> $GITHUB_ENV + echo "FC=nvfortran" >> $GITHUB_ENV + echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 8ded5a0779f718c0c1aa191e7fdbd8790f480d82 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:06:37 +0000 Subject: [PATCH 055/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a0f857c21..7921e29b8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -125,8 +125,8 @@ jobs: ####################################################################################### - name: Environment for nvidia compiler. run: | - echo "CC=nvc" >> $GITHUB_ENV - echo "FC=nvfortran" >> $GITHUB_ENV + echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV From d06dd31ad1a2b5bddab2f82162594f1ca80e6093 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:15:20 +0000 Subject: [PATCH 056/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7921e29b8..515d2d08c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -130,15 +130,15 @@ jobs: echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV -# - name: Cache bacio library v2.4.1 -# id: cache-bacio-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/bacio -# key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key + - name: Cache bacio library v2.4.1 + id: cache-bacio-fortran + uses: actions/cache@v3 + with: + path: /home/runner/bacio + key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key - name: Install bacio library v2.4.1 -# if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' + if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio cd bacio && mkdir build && cd build @@ -147,15 +147,15 @@ jobs: make install echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV -# - name: Cache SP-library v2.3.3 -# id: cache-sp-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/NCEPLIBS-sp -# key: cache-sp-fortran-${{matrix.fortran-compiler}}-key + - name: Cache SP-library v2.3.3 + id: cache-sp-fortran + uses: actions/cache@v3 + with: + path: /home/runner/NCEPLIBS-sp + key: cache-sp-fortran-${{matrix.fortran-compiler}}-key - name: Install SP-library v2.3.3 -# if: steps.cache-sp-fortran.outputs.cache-hit != 'true' + if: steps.cache-sp-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp cd NCEPLIBS-sp && mkdir build && cd build @@ -164,15 +164,15 @@ jobs: make install echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV -# - name: Cache w3emc library v2.9.2 -# id: cache-w3emc-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/myw3emc -# key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key + - name: Cache w3emc library v2.9.2 + id: cache-w3emc-fortran + uses: actions/cache@v3 + with: + path: /home/runner/myw3emc + key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key - name: Install w3emc library v2.9.2 -# if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' + if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc cd NCEPLIBS-w3emc && mkdir build && cd build From 52a22f2f408cafb2ae26d93f7c0b629843723153 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:25:05 +0000 Subject: [PATCH 057/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 515d2d08c..c5fed1110 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -78,16 +78,14 @@ jobs: - name: Add conda to system path run: | echo $CONDA/bin >> $GITHUB_PATH - df -h - name: Install NetCDF Python libraries run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - df -h + - name: Update system packages run: | sudo apt-get update - df -h ####################################################################################### # Install Nvidia. @@ -123,7 +121,7 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for nvidia compiler. + - name: Set environment for nvidia compiler. run: | echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV @@ -180,3 +178,25 @@ jobs: make -j2 make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + + - name: Install NetCDF C library + run: | + sudo apt-get update + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib \ No newline at end of file From fe42708a2dcafce60e0523145b2fd91d78749781 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:39:20 +0000 Subject: [PATCH 058/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c5fed1110..8493f8d25 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -192,6 +192,8 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library + env: + FCFLAGS: -fPIC if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git @@ -199,4 +201,27 @@ jobs: ./configure make -j sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib \ No newline at end of file + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + + ####################################################################################### + # Build SCM. + ####################################################################################### + + - name: Configure build with CMake (Release) + if: contains(matrix.build-type, 'Release') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} ../src + + - name: Configure build with CMake (Debug) + if: contains(matrix.build-type, 'Debug') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + + - name: Build SCM + run: | + cd ${SCM_ROOT}/scm/bin + make -j4 \ No newline at end of file From a6955aba104af1d864e103d234d22ef6b4b66254 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:40:50 +0000 Subject: [PATCH 059/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8493f8d25..c26b6c884 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -212,7 +212,7 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From c94ba5a011de6836d22a020f283041af02a48378 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:50:10 +0000 Subject: [PATCH 060/189] Update CMakeLists for Nvidia support --- scm/src/CMakeLists.txt | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index b952680b6..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,9 +30,6 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) -if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") - message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) -endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -129,7 +126,8 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") @@ -186,6 +184,37 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + +elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + if(DEFINED ENABLE_NVIDIA_OPENACC) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + endif() + + if(NOT 32BIT) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") + endif() + + if (${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -g") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + endif() + + set(MPI_C_COMPILER mpicc) + set(MPI_CXX_COMPILER mpicxx) + set(MPI_Fortran_COMPILER mpif90) + + set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 7ff6081afa39c3ffff0866a140769e474be62f77 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:22:57 +0000 Subject: [PATCH 061/189] Update CMakeLists for Nvidia support --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c26b6c884..fc5250517 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,6 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release]#, Debug] + enable-gpu-acc: [False]#, True] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -111,6 +112,7 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" @@ -123,6 +125,14 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran + echo "Removing tarball" + rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV @@ -212,7 +222,10 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src + if: contains(matrix.enable-gpu-acc, 'True') + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + else: + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From b448eb40aeacd0487e86046cd453d3493e3f9e47 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:24:20 +0000 Subject: [PATCH 062/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index fc5250517..f19be132a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -128,7 +128,7 @@ jobs: export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" - nvfortran --version + nvfortran --version echo "The path to nvfortran is:" command -v nvfortran echo "Removing tarball" From b268880a562401183616791c5b727178d73205ce Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:26:16 +0000 Subject: [PATCH 063/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f19be132a..3b25cf502 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -112,7 +112,7 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" @@ -126,7 +126,7 @@ jobs: - name: Set environment for nvidia compiler. run: | export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From f18a65a6bb6761eb984acc6adf5853b74f05ae86 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:48:45 +0000 Subject: [PATCH 064/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3b25cf502..3de8f3438 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -91,15 +91,15 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: Cache Nvidia HPC - id: cache-nvidia - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk - key: cache-nvidia-key +# - name: Cache Nvidia HPC +# id: cache-nvidia +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk +# key: cache-nvidia-key - name: Nvidia setup compilers - if: steps.cache-nvidia.outputs.cache-hit != 'true' +# if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -125,14 +125,14 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran - echo "Removing tarball" - rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + #export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + #export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + #echo "The nvfortran installed is:" + #nvfortran --version + #echo "The path to nvfortran is:" + #command -v nvfortran + #echo "Removing tarball" + #rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV From af05846982d3f8819f354c9d7798cc474bd373c6 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 22:02:54 +0000 Subject: [PATCH 065/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3de8f3438..b5799cb71 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -222,10 +222,11 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: contains(matrix.enable-gpu-acc, 'True') - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True - else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src +# if: contains(matrix.enable-gpu-acc, 'True') +# cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True +# else: +# cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 47f0a6cbe6e70a38885cd36065c264e95a2e0339 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 22:50:47 +0000 Subject: [PATCH 066/189] Updated CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b5799cb71..1adfa85d1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -189,6 +189,11 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + - name: Install Curl + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev + - name: Install NetCDF C library run: | sudo apt-get update From d2c04ed938763697ebff9f5f158f96aaaf58243e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 22:33:46 +0000 Subject: [PATCH 067/189] Update cmakelists --- scm/src/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..4ac9f87c9 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -68,6 +68,9 @@ find_package(NetCDF REQUIRED COMPONENTS C Fortran) find_package(bacio REQUIRED) find_package(sp REQUIRED) find_package(w3emc REQUIRED) +if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + find_package(curl REQUIRED) +endif() SET(CCPP_FRAMEWORK_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/framework) SET(CCPP_PHYSICS_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/physics) @@ -280,6 +283,9 @@ TARGET_LINK_LIBRARIES(scm sp::sp_d) TARGET_LINK_LIBRARIES(scm w3emc::w3emc_d) TARGET_LINK_LIBRARIES(scm ccpp_framework) TARGET_LINK_LIBRARIES(scm ccpp_physics) +if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + TARGET_LINK_LIBRARIES(scm curl) +endif() SET_TARGET_PROPERTIES(scm PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}" From 90b7fea11dd7ea67a8bfafecb55d0e157d29bcf0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 22:44:36 +0000 Subject: [PATCH 068/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- scm/src/CMakeLists.txt | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 1adfa85d1..c671b50b6 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -84,9 +84,9 @@ jobs: run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - - name: Update system packages - run: | - sudo apt-get update +# - name: Update system packages +# run: | +# sudo apt-get update ####################################################################################### # Install Nvidia. diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 4ac9f87c9..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -68,9 +68,6 @@ find_package(NetCDF REQUIRED COMPONENTS C Fortran) find_package(bacio REQUIRED) find_package(sp REQUIRED) find_package(w3emc REQUIRED) -if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - find_package(curl REQUIRED) -endif() SET(CCPP_FRAMEWORK_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/framework) SET(CCPP_PHYSICS_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/physics) @@ -283,9 +280,6 @@ TARGET_LINK_LIBRARIES(scm sp::sp_d) TARGET_LINK_LIBRARIES(scm w3emc::w3emc_d) TARGET_LINK_LIBRARIES(scm ccpp_framework) TARGET_LINK_LIBRARIES(scm ccpp_physics) -if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - TARGET_LINK_LIBRARIES(scm curl) -endif() SET_TARGET_PROPERTIES(scm PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}" From ddb35914f55932f0f28729da18f71b4b1cf75596 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 23:00:16 +0000 Subject: [PATCH 069/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c671b50b6..577c39224 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,7 +51,7 @@ jobs: android: true dotnet: true haskell: true - large-packages: true + large-packages: false docker-images: true swap-storage: true From 7ef27e803c94af3cd260eb5f1d52f624b96cb7e9 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 23:09:44 +0000 Subject: [PATCH 070/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 577c39224..ad1b0e909 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,9 +51,9 @@ jobs: android: true dotnet: true haskell: true - large-packages: false + large-packages: true docker-images: true - swap-storage: true + swap-storage: false - name: Check space (post) run: | @@ -125,14 +125,6 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | - #export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - #export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - #echo "The nvfortran installed is:" - #nvfortran --version - #echo "The path to nvfortran is:" - #command -v nvfortran - #echo "Removing tarball" - #rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV From dbc4334bdfa2101e959bbeee3c6fb2c096ccf14e Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 08:33:15 -0700 Subject: [PATCH 071/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ad1b0e909..9fddf3364 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -48,11 +48,11 @@ jobs: # all of these default to true, but feel free to set to # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true + android: false + dotnet: false + haskell: false + large-packages: false + docker-images: false swap-storage: false - name: Check space (post) From a38f5c0548369b641c6ee16aafe4580d387be700 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:13:37 -0700 Subject: [PATCH 072/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9fddf3364..78e2d8022 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,7 +51,7 @@ jobs: android: false dotnet: false haskell: false - large-packages: false + large-packages: true docker-images: false swap-storage: false From 3f79deaa700c13960e1616156fff59f0d9929763 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:36:31 -0700 Subject: [PATCH 073/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 78e2d8022..440ea688f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -50,7 +50,7 @@ jobs: # "false" if necessary for your workflow android: false dotnet: false - haskell: false + haskell: true large-packages: true docker-images: false swap-storage: false From 008903113e5fe479dc8b53488c801818405ed449 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:56:06 -0700 Subject: [PATCH 074/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 440ea688f..6fe53375c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -105,7 +105,7 @@ jobs: NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk - NVHPC_INSTALL_TYPE: single + NVHPC_INSTALL_TYPE: auto run: | mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz @@ -181,10 +181,10 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - name: Install Curl - run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev +# - name: Install Curl +# run: | +# sudo apt-get update +# sudo apt-get install libcurl4-openssl-dev - name: Install NetCDF C library run: | From 8031de348c8c073bf4baed816a74868b001212d5 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:07:30 -0700 Subject: [PATCH 075/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6fe53375c..bc5357169 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -105,7 +105,8 @@ jobs: NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk - NVHPC_INSTALL_TYPE: auto + NVHPC_INSTALL_TYPE: network + NVHPC_INSTALL_LOCAL_DIR: /home/runner/hpc_sdk run: | mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 789711cbe21bc41a5ebbf418dadb8d1a5c2cfef7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:24:08 -0700 Subject: [PATCH 076/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bc5357169..4f0e56c57 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -121,6 +121,10 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + - name: Call uname command + run: | + `uname -s`_`uname -m` + ####################################################################################### # Install FORTRAN dependencies ####################################################################################### From 28267f8dd50cd7514598545ff273587ac7bbe5a3 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:26:21 -0700 Subject: [PATCH 077/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4f0e56c57..cf48b38af 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -123,7 +123,8 @@ jobs: - name: Call uname command run: | - `uname -s`_`uname -m` + uname -s + uname -m ####################################################################################### # Install FORTRAN dependencies From 157490010cb970de896d46d1046e76e05bebf324 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:28:39 -0700 Subject: [PATCH 078/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cf48b38af..1e7fcfadb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -58,6 +58,8 @@ jobs: - name: Check space (post) run: | df -h + uname -c + uname -s ####################################################################################### # Initial @@ -121,11 +123,6 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - - name: Call uname command - run: | - uname -s - uname -m - ####################################################################################### # Install FORTRAN dependencies ####################################################################################### From 923fa461496ce54e1872ea8b887bd1996b4d6d9d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:32:56 -0700 Subject: [PATCH 079/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 1e7fcfadb..388bbedf0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -38,6 +38,8 @@ jobs: - name: Check space (pre) run: | df -h + uname -s + uname -m - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -58,8 +60,6 @@ jobs: - name: Check space (post) run: | df -h - uname -c - uname -s ####################################################################################### # Initial From 9768b94748065a114230fc2ca4d6a949fa2e1fbb Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 12:52:45 -0700 Subject: [PATCH 080/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 388bbedf0..ec3c84dac 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -38,8 +38,6 @@ jobs: - name: Check space (pre) run: | df -h - uname -s - uname -m - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -184,8 +182,10 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV -# - name: Install Curl -# run: | + - name: Install Curl + run: | + sudo apt-get install curl + sudo apt-get install libssl-dev libcurl4-openssl-dev # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev From e93fb0329f73fbdceaa9ecc1519471b887b541a0 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:26:35 -0700 Subject: [PATCH 081/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ec3c84dac..cc06ef771 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,11 +17,13 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran - NFVERSION: v4.5.3 + NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + zlib_ROOT: /home/runner/zlib + HDF5_ROOT: /home/runner/hdf5 suites: SCM_RAP F90FLAGS: -O3 CC: nvc @@ -131,6 +133,32 @@ jobs: echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + - name: Install zlib + env: + CFLAGS: -fPIC + run: | + wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz + tar -zxvf zlib-1.2.13.tar.gz + cd zlib-1.2.13 + ./configure --prefix=${ZLIB_ROOT} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ZLIB_ROOT}/lib + + - name: Install HDF5 + env: + CPPFLAGS: -I${ZLIB_ROOT}/include + LDFLAGS: -L${ZLIB_ROOT}/lib + run: | + wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz + tar -zxvf hdf5-1_14_1-2.tar.gz + cd hdf5-hdf5-1_14_1-2 + ./configure --prefix=${HDF5_ROOT} --with-zlib=${ZLIB_ROOT} + make -j4 + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib + export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -191,6 +219,9 @@ jobs: - name: Install NetCDF C library run: | +# wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz +# tar -zvxf v4.7.4.tar.gz +# cd netcdf-c-4.7.4 sudo apt-get update sudo apt-get install libnetcdf-dev From 74428b978eba07b779d86fcdcd01cb01517c4410 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:27:37 -0700 Subject: [PATCH 082/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cc06ef771..c7f97f6ea 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -158,7 +158,7 @@ jobs: make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib - + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 From 2f6c1660abd8e42308484a9e48ab8c8e8f7ab8a8 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:28:27 -0700 Subject: [PATCH 083/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c7f97f6ea..360fb9bcc 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -210,18 +210,15 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - name: Install Curl - run: | - sudo apt-get install curl - sudo apt-get install libssl-dev libcurl4-openssl-dev +# - name: Install Curl +# run: | +# sudo apt-get install curl +# sudo apt-get install libssl-dev libcurl4-openssl-dev # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev - name: Install NetCDF C library run: | -# wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz -# tar -zvxf v4.7.4.tar.gz -# cd netcdf-c-4.7.4 sudo apt-get update sudo apt-get install libnetcdf-dev From 5e97c33e70e2fd78ac7c2d838e7b1a33b65d86ab Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:55:51 -0700 Subject: [PATCH 084/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 360fb9bcc..535e1383d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -140,20 +140,20 @@ jobs: wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz cd zlib-1.2.13 - ./configure --prefix=${ZLIB_ROOT} + ./configure --prefix=${zlib_ROOT} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ZLIB_ROOT}/lib + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib - name: Install HDF5 env: - CPPFLAGS: -I${ZLIB_ROOT}/include - LDFLAGS: -L${ZLIB_ROOT}/lib + CPPFLAGS: -I${zlib_ROOT}/include + LDFLAGS: -L${zlib_ROOT}/lib run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz cd hdf5-hdf5-1_14_1-2 - ./configure --prefix=${HDF5_ROOT} --with-zlib=${ZLIB_ROOT} + ./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT} make -j4 make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib From 9286df0eba8d268bc030cf1d8c8640cb4d6cfb7c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 14:38:33 -0700 Subject: [PATCH 085/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 535e1383d..cf7877460 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,6 +17,7 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran + NCHOME: /home/runner/netcdf NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp @@ -159,6 +160,41 @@ jobs: export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Install NetCDF C library + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + run: | + wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz + tar -zvxf v4.7.4.tar.gz + cd netcdf-c-4.7.4 + ./configure --prefix=${NCHOME} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NCHOME}/lib + export PATH=${PATH}:${NCHOME}/lib +# sudo apt-get update +# sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + env: + FCFLAGS: -fPIC + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -217,30 +253,6 @@ jobs: # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev - - name: Install NetCDF C library - run: | - sudo apt-get update - sudo apt-get install libnetcdf-dev - - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - - - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib - ####################################################################################### # Build SCM. ####################################################################################### From fa714f42a4883a4de14def1c43fe2d7e933da0da Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 14:49:02 -0700 Subject: [PATCH 086/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cf7877460..92678ae3a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -160,6 +160,11 @@ jobs: export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Install Curl + run: | + sudo apt-get install curl + sudo apt-get install libssl-dev libcurl4-openssl-dev + - name: Install NetCDF C library env: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include From de7cde1e721ead31a62e3362b71a8789a03fa286 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:10:54 -0700 Subject: [PATCH 087/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 92678ae3a..d574af144 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,7 +17,7 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran - NCHOME: /home/runner/netcdf + NETCDF: /home/runner/netcdf NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp @@ -170,35 +170,39 @@ jobs: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib run: | + echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -I/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - ./configure --prefix=${NCHOME} + ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NCHOME}/lib - export PATH=${PATH}:${NCHOME}/lib -# sudo apt-get update -# sudo apt-get install libnetcdf-dev + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib + export PATH=${PATH}:${NETCDF}/lib - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache NetCDF Fortran library +# id: cache-netcdf-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/netcdf-fortran +# key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library env: FCFLAGS: -fPIC - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + FFLAGS: -fPIC + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib +# if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz + tar -zvxf v4.6.1.tar.gz + cd netcdf-fortran-4.6.1 + ./configure --prefix=${NETCDF} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - name: Cache bacio library v2.4.1 id: cache-bacio-fortran From 22f18fcae854cc5784d9c7d971562e347f8a2a86 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:12:37 -0700 Subject: [PATCH 088/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d574af144..eaad201d8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -200,7 +200,7 @@ jobs: tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 ./configure --prefix=${NETCDF} - make + make make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib From 990e36bcaf5b9bcac97e766bb161e961a5d8826c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:15:30 -0700 Subject: [PATCH 089/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index eaad201d8..332700b6b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -171,7 +171,7 @@ jobs: LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib run: | echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -I/home/runner/zlib/lib" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From bf45ad5dddbcab7e18a6be415c0c0927eb172f9c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:32:13 -0700 Subject: [PATCH 090/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 332700b6b..ab3646d8f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,6 +31,7 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: From f6673e7ba9c60a411a220b2be16f223e6750e261 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:41:02 -0700 Subject: [PATCH 091/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ab3646d8f..93b624901 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,7 +31,7 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran - LD_LIBRARY_PATH: ${{ github.workspace }} +# LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: @@ -145,7 +145,8 @@ jobs: ./configure --prefix=${zlib_ROOT} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib + #export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib + echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Install HDF5 env: @@ -158,8 +159,10 @@ jobs: ./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT} make -j4 make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib - export PATH=${PATH}:${HDF5_ROOT}/lib + echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV +# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib +# export PATH=${PATH}:${HDF5_ROOT}/lib - name: Install Curl run: | @@ -179,8 +182,10 @@ jobs: ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - export PATH=${PATH}:${NETCDF}/lib + echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV +# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib +# export PATH=${PATH}:${NETCDF}/lib # - name: Cache NetCDF Fortran library # id: cache-netcdf-fortran @@ -203,7 +208,6 @@ jobs: ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - name: Cache bacio library v2.4.1 id: cache-bacio-fortran From 8f57ff042c53bb7808f0de73ff53b515a020f1f7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:57:37 -0700 Subject: [PATCH 092/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 93b624901..4228863b3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -179,7 +179,7 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From 407280d2f121d184d3425847d970883c2a9aaef7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 16:12:50 -0700 Subject: [PATCH 093/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4228863b3..0dd82959d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -205,7 +205,7 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} make make install From 3d10750f2466f7902a9f9a29027dde5ef8c3f1cb Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:24:56 -0700 Subject: [PATCH 094/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 61 ++++++++----------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 0dd82959d..4c2f8ae30 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -88,22 +88,18 @@ jobs: run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml -# - name: Update system packages -# run: | -# sudo apt-get update - ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: Cache Nvidia HPC -# id: cache-nvidia -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk -# key: cache-nvidia-key + - name: Cache Nvidia HPC + id: cache-nvidia + uses: actions/cache@v3 + with: + path: /home/runner/hpc_sdk + key: cache-nvidia-key - name: Nvidia setup compilers -# if: steps.cache-nvidia.outputs.cache-hit != 'true' + if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -161,18 +157,24 @@ jobs: make install echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV -# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib -# export PATH=${PATH}:${HDF5_ROOT}/lib - name: Install Curl run: | sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev + - name: Cache NetCDF C library + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-c + key: cache-netcdf-c-${{matrix.fortran-compiler}}-key + - name: Install NetCDF C library env: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV @@ -184,23 +186,21 @@ jobs: make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV -# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib -# export PATH=${PATH}:${NETCDF}/lib -# - name: Cache NetCDF Fortran library -# id: cache-netcdf-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/netcdf-fortran -# key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - FFLAGS: -fPIC - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib -# if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' +# env: +# FCFLAGS: -fPIC +# FFLAGS: -fPIC +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz @@ -260,13 +260,6 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV -# - name: Install Curl -# run: | -# sudo apt-get install curl -# sudo apt-get install libssl-dev libcurl4-openssl-dev -# sudo apt-get update -# sudo apt-get install libcurl4-openssl-dev - ####################################################################################### # Build SCM. ####################################################################################### From cf8830f1926fdf6f7696019dd62e284604714dee Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:32:48 -0700 Subject: [PATCH 095/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4c2f8ae30..c2a31d60d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,7 +31,6 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran -# LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: @@ -91,15 +90,8 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: Cache Nvidia HPC - id: cache-nvidia - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk - key: cache-nvidia-key - name: Nvidia setup compilers - if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 From bb798ac1a60f13bfb9e9ba0a7024327e01dc754d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:48:22 -0700 Subject: [PATCH 096/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c2a31d60d..d6c9432ba 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -91,7 +91,7 @@ jobs: # Install Nvidia. ####################################################################################### - - name: Nvidia setup compilers + - name: Nvidia setup compilers. env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -113,19 +113,29 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - ####################################################################################### - # Install FORTRAN dependencies - ####################################################################################### - - name: Set environment for nvidia compiler. + - name: Set environment for Nvidia compiler. run: | echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + ####################################################################################### + # Install FORTRAN dependencies + ####################################################################################### + + - name: Cache Zlib + run: | + id: cache-zlib + uses: actions/cache@v3 + with: + path: /home/runner/zlib + key: cache-zlib-${{matrix.fortran-compiler}}-key + - name: Install zlib env: CFLAGS: -fPIC + if: steps.cache-zlib.outputs.cache-hit != 'true' run: | wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz @@ -133,13 +143,21 @@ jobs: ./configure --prefix=${zlib_ROOT} make make install - #export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Cache HDF5 + run: | + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: /home/runner/hdf5 + key: cache-hdf5-${{matrix.fortran-compiler}}-key + - name: Install HDF5 env: CPPFLAGS: -I${zlib_ROOT}/include LDFLAGS: -L${zlib_ROOT}/lib + if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz @@ -163,13 +181,13 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib +# env: +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | - echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV +# echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV +# echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From 398a2920c63cd38f6b86432c565a2926821a828d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:49:32 -0700 Subject: [PATCH 097/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d6c9432ba..f2c14142e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -131,7 +131,7 @@ jobs: with: path: /home/runner/zlib key: cache-zlib-${{matrix.fortran-compiler}}-key - + - name: Install zlib env: CFLAGS: -fPIC From 6bc8277b81c8312fc4c396f153f1563505578a18 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:52:02 -0700 Subject: [PATCH 098/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f2c14142e..e5e155e82 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -173,12 +173,12 @@ jobs: sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev - - name: Cache NetCDF C library - id: cache-netcdf-c - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-c - key: cache-netcdf-c-${{matrix.fortran-compiler}}-key +# - name: Cache NetCDF C library +# id: cache-netcdf-c +# uses: actions/cache@v3 +# with: +# path: /home/runner/netcdf-c +# key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library # env: From 24e091e976bceb782b9a4078c33863fab18647bc Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:53:33 -0700 Subject: [PATCH 099/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e5e155e82..4d7fe545f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -173,17 +173,17 @@ jobs: sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev -# - name: Cache NetCDF C library -# id: cache-netcdf-c -# uses: actions/cache@v3 -# with: -# path: /home/runner/netcdf-c -# key: cache-netcdf-c-${{matrix.fortran-compiler}}-key + - name: Cache NetCDF C library + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-c + key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library -# env: -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | # echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV From b1ce93cc3b208436f7ff530164e5da4976d3459a Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:55:17 -0700 Subject: [PATCH 100/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4d7fe545f..cc9384522 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -181,13 +181,11 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib +# env: +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | -# echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV -# echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From 9df5652aa001a0b7f96a398b48c8196fe5d78c2d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:57:10 -0700 Subject: [PATCH 101/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cc9384522..2735f2428 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -181,11 +181,13 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library -# env: -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | + echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From fb7f34f3c52965bb6f1627680ea42a4cdeb913e1 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:59:57 -0700 Subject: [PATCH 102/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 2735f2428..9c6d988b1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -205,11 +205,11 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library -# env: -# FCFLAGS: -fPIC -# FFLAGS: -fPIC -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib + env: + FCFLAGS: -fPIC + FFLAGS: -fPIC + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz From af52b7647ec3b8e272394d488f018e70d5e3c95c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:04:18 -0700 Subject: [PATCH 103/189] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9c6d988b1..c85b1693d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -124,14 +124,6 @@ jobs: # Install FORTRAN dependencies ####################################################################################### - - name: Cache Zlib - run: | - id: cache-zlib - uses: actions/cache@v3 - with: - path: /home/runner/zlib - key: cache-zlib-${{matrix.fortran-compiler}}-key - - name: Install zlib env: CFLAGS: -fPIC @@ -145,14 +137,6 @@ jobs: make install echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - name: Cache HDF5 - run: | - id: cache-hdf5 - uses: actions/cache@v3 - with: - path: /home/runner/hdf5 - key: cache-hdf5-${{matrix.fortran-compiler}}-key - - name: Install HDF5 env: CPPFLAGS: -I${zlib_ROOT}/include From 035a7659c82af4b2a3e4f9445e0e81afa97e7134 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:08:34 -0700 Subject: [PATCH 104/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c85b1693d..4c2596293 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -165,13 +165,8 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | - echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From d7a6bff9b9f0f874e6e2bccb3ed43614a56cf8b0 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:09:36 -0700 Subject: [PATCH 105/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4c2596293..84c3ab44e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -184,11 +184,6 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - FFLAGS: -fPIC - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz From fbf58568a498bddcfa547a5d222f93ca4ff6ac3f Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:10:19 -0700 Subject: [PATCH 106/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 84c3ab44e..5b6401446 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -170,7 +170,9 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" + LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" + ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From 897cfca7d910cc4f07fc56622ae678b34641aa18 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:11:54 -0700 Subject: [PATCH 107/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5b6401446..358a68501 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -191,7 +191,10 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" + LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" + ./configure --prefix=${NETCDF} make make install From 3eab6709382c48437e28ae7ec9b6f8705ddb6008 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:16:11 -0700 Subject: [PATCH 108/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 358a68501..5c14d1371 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,12 +25,12 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_RAP - F90FLAGS: -O3 - CC: nvc - FC: nvfortran - CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc - CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 +# F90FLAGS: -O3 +# CC: nvc +# FC: nvfortran +# CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc +# CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # Workflow steps steps: From f030d722eb2a346cec52dcef5be4f3d9baedc71e Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:21:19 -0700 Subject: [PATCH 109/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5c14d1371..45ef63bb3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -274,4 +274,15 @@ jobs: - name: Build SCM run: | cd ${SCM_ROOT}/scm/bin - make -j4 \ No newline at end of file + make -j4 + + - name: Download data for SCM + run: | + cd ${SCM_ROOT} + ./contrib/get_all_static_data.sh + ./contrib/get_thompson_tables.sh + + - name: Run SCM RTs + run: | + cd ${SCM_ROOT}/scm/bin + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 From 442f3546874a6ed73c8cc2f1b395747bf5e1c62f Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:24:01 -0700 Subject: [PATCH 110/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 45ef63bb3..40ea03bfb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -170,9 +170,7 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" - LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" - ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV @@ -191,10 +189,7 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - FCFLAGS="-fPIC" FFLAGS="-fPIC" - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" - LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" - ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} make make install From b901cd40d8b8207d3ce7628530a0d82a26463e8b Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:42:43 -0700 Subject: [PATCH 111/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 40ea03bfb..74ace14c7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,12 +25,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 -# F90FLAGS: -O3 -# CC: nvc -# FC: nvfortran -# CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc -# CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + suites: SCM_RAP,SCM_RRFS_v1beta # Workflow steps steps: From 7970fdc6cee30f73b454e094b96571816053bf59 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:58:17 -0700 Subject: [PATCH 112/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 74ace14c7..f58c174e5 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -275,4 +275,4 @@ jobs: - name: Run SCM RTs run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v From 09928ca41cabc5532aba2b45fe532696d174372e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 17:40:14 +0000 Subject: [PATCH 113/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f58c174e5..731f7f072 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release]#, Debug] + build-type: [Release, Debug] enable-gpu-acc: [False]#, True] py-version: [3.7.13]#, 3.9.12] @@ -275,4 +275,4 @@ jobs: - name: Run SCM RTs run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v From 5a461157ff391a69b346a5acc66e41266a218da4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 17:59:55 +0000 Subject: [PATCH 114/189] Add file for Nvidia RTS --- test/rt_test_cases_nvidia.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/rt_test_cases_nvidia.py diff --git a/test/rt_test_cases_nvidia.py b/test/rt_test_cases_nvidia.py new file mode 100644 index 000000000..a19c81ba9 --- /dev/null +++ b/test/rt_test_cases_nvidia.py @@ -0,0 +1,9 @@ +run_list = [\ + #---------------------------------------------------------------------------------------------------------------------------------------------- + # CCPP-SCM v6 supported suites + #---------------------------------------------------------------------------------------------------------------------------------------------- + {"case": "arm_sgp_summer_1997_A", "suite": "SCM_RAP"}, \ + {"case": "twpice", "suite": "SCM_RAP"}, \ + {"case": "bomex", "suite": "SCM_RAP"}, \ + {"case": "astex", "suite": "SCM_RAP"}, \ + {"case": "LASSO_2016051812", "suite": "SCM_RAP"}] From f89b1d6f8747dd453ffd8a0336044a3e7311d34d Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:07:20 +0000 Subject: [PATCH 115/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 731f7f072..7fc467a0f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,7 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release, Debug] - enable-gpu-acc: [False]#, True] + enable-gpu-acc: [false, true] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -248,11 +248,11 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src -# if: contains(matrix.enable-gpu-acc, 'True') -# cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True -# else: -# cmake -DCCPP_SUITES=${suites} ../src +# cmake -DCCPP_SUITES=${suites} ../src + if: matrix.enable-gpu-acc + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + else: + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 51b7baa992ae844a1c2620fbd309babd1e024259 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:08:32 +0000 Subject: [PATCH 116/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7fc467a0f..8e0c1bded 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -248,9 +248,8 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin -# cmake -DCCPP_SUITES=${suites} ../src - if: matrix.enable-gpu-acc - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + if: matrix.enable-gpu-acc + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True else: cmake -DCCPP_SUITES=${suites} ../src From d712d560e994c30337540f27bcd9fb3dc3dec983 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:14:44 +0000 Subject: [PATCH 117/189] Revert change to CMakeLists --- scm/src/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..d45360a15 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,6 +30,9 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) +if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") + message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) +endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -126,8 +129,7 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") From 0b2ee57118ee9c8ef1e747b604ae9546e68ec5aa Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:16:23 +0000 Subject: [PATCH 118/189] Revert change to CMakeLists --- scm/src/CMakeLists.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index d45360a15..705fa2ae1 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -208,15 +208,6 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") endif() - set(MPI_C_COMPILER mpicc) - set(MPI_CXX_COMPILER mpicxx) - set(MPI_Fortran_COMPILER mpif90) - - set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) - else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 16c79388384d23ec917054b2bff32e20d09afc60 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:30:03 +0000 Subject: [PATCH 119/189] Revert change to CMakeLists --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8e0c1bded..37cd360f0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -248,10 +248,10 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: matrix.enable-gpu-acc - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + if: contains(matrix.enable-gpu-acc, 'true') + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 9fe09eb53c641f6d8ef3b619bba2ed80a883ddef Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:36:27 +0000 Subject: [PATCH 120/189] Revert change to CMakeLists --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 37cd360f0..eb4eed698 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: @@ -122,7 +122,6 @@ jobs: - name: Install zlib env: CFLAGS: -fPIC - if: steps.cache-zlib.outputs.cache-hit != 'true' run: | wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz @@ -136,7 +135,6 @@ jobs: env: CPPFLAGS: -I${zlib_ROOT}/include LDFLAGS: -L${zlib_ROOT}/lib - if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz @@ -272,6 +270,7 @@ jobs: ./contrib/get_thompson_tables.sh - name: Run SCM RTs + if: contains(matrix.enable-gpu-acc, 'false') run: | cd ${SCM_ROOT}/scm/bin ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v From 659d780b1bfb0dd8157450eca6926d25478a163a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:44:20 +0000 Subject: [PATCH 121/189] Update CI --- scm/src/CMakeLists.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 705fa2ae1..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,9 +30,6 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) -if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") - message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) -endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -129,7 +126,8 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") @@ -208,6 +206,15 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") endif() + set(MPI_C_COMPILER mpicc) + set(MPI_CXX_COMPILER mpicxx) + set(MPI_Fortran_COMPILER mpif90) + + set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 229473c55406bbf001c06f7ff98c1b209392a1cd Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:53:04 +0000 Subject: [PATCH 122/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index eb4eed698..9ace68508 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,7 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release, Debug] - enable-gpu-acc: [false, true] + enable-gpu-acc: [False, True] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -246,17 +246,14 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: contains(matrix.enable-gpu-acc, 'true') - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True - else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM run: | From ba034f497f92080333ff3633b7b89fa4777815b6 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:54:31 +0000 Subject: [PATCH 123/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9ace68508..c3f88d40c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] jobs: From 59b73cdf29f75325a7827dd283466656f1c62515 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:04:53 +0000 Subject: [PATCH 124/189] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 14 +++----------- scm/src/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c3f88d40c..9f33b0803 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -241,19 +241,11 @@ jobs: # Build SCM. ####################################################################################### - - name: Configure build with CMake (Release) - if: contains(matrix.build-type, 'Release') + - name: Configure build with CMake run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} - - - name: Configure build with CMake (Debug) - if: contains(matrix.build-type, 'Debug') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM run: | @@ -267,7 +259,7 @@ jobs: ./contrib/get_thompson_tables.sh - name: Run SCM RTs - if: contains(matrix.enable-gpu-acc, 'false') + if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..10885cf28 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -186,7 +186,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - if(DEFINED ENABLE_NVIDIA_OPENACC) + if(ENABLE_NVIDIA_OPENACC MATCHES "True") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") else() From c12544ce21dee4fa6e2cd30c2ec43f85fd4d47a4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:18:40 +0000 Subject: [PATCH 125/189] Update CI --- scm/src/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 10885cf28..3cfe66673 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,6 +30,9 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) +if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") + message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) +endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -126,8 +129,7 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") From f82f995044ebcd78b73404c8cdd06b009c6725c2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:40:04 +0000 Subject: [PATCH 126/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9f33b0803..19c13ee06 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,4 +1,4 @@ -name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers +name: CI test to build the CCPP-SCM on ubuntu v22.04 on: [push,pull_request,workflow_dispatch] @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] + build-type: [Release] enable-gpu-acc: [False, True] py-version: [3.7.13]#, 3.9.12] @@ -247,7 +247,7 @@ jobs: mkdir bin && cd bin cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - - name: Build SCM + - name: Build SCM. run: | cd ${SCM_ROOT}/scm/bin make -j4 @@ -258,7 +258,7 @@ jobs: ./contrib/get_all_static_data.sh ./contrib/get_thompson_tables.sh - - name: Run SCM RTs + - name: Run SCM RTs (w/o GPU) if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin From 8d6a1b553a06ffc1c5daf77c9de9bbbfa35d3ef8 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 22:05:23 +0000 Subject: [PATCH 127/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 19c13ee06..bace35b9d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -12,7 +12,7 @@ jobs: fortran-compiler: [nvfortran] build-type: [Release] enable-gpu-acc: [False, True] - py-version: [3.7.13]#, 3.9.12] + py-version: [3.7.13, 3.9.12] # Environmental variables env: @@ -25,7 +25,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_RAP,SCM_RRFS_v1beta + suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 # Workflow steps steps: From 9521536894f7d407e504672c10fe09a76f258bd5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:17:56 +0000 Subject: [PATCH 128/189] Update CI --- scm/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 3cfe66673..2ef20bb84 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -188,7 +188,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - if(ENABLE_NVIDIA_OPENACC MATCHES "True") + if(ENABLE_NVIDIA_OPENACC MATCHES "true") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") else() From 999d6cba8677e22036f2b614a1b19044a49e6427 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:24:11 +0000 Subject: [PATCH 129/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bace35b9d..c4e498cf2 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,15 +10,13 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release] + build-type: [Release, Debug] enable-gpu-acc: [False, True] py-version: [3.7.13, 3.9.12] # Environmental variables env: - NFHOME: /home/runner/netcdf-fortran NETCDF: /home/runner/netcdf - NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc @@ -238,7 +236,7 @@ jobs: echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV ####################################################################################### - # Build SCM. + # Build and run SCM regression tests (ccpp-scm/test/rt_test_cases.py) ####################################################################################### - name: Configure build with CMake @@ -253,6 +251,7 @@ jobs: make -j4 - name: Download data for SCM + if: contains(matrix.enable-gpu-acc, 'True') run: | cd ${SCM_ROOT} ./contrib/get_all_static_data.sh @@ -262,4 +261,4 @@ jobs: if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v From 4d71d1ec518eb579813c80d6d0167aff9e1b8736 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:41:50 +0000 Subject: [PATCH 130/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c4e498cf2..5a1c4ebef 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] + build-type: [Release]#, Debug] enable-gpu-acc: [False, True] py-version: [3.7.13, 3.9.12] From 208bed3dce56555942965adc979c6fd8f7a0206a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 14:21:50 +0000 Subject: [PATCH 131/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5a1c4ebef..7bfaa4bcf 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -251,7 +251,6 @@ jobs: make -j4 - name: Download data for SCM - if: contains(matrix.enable-gpu-acc, 'True') run: | cd ${SCM_ROOT} ./contrib/get_all_static_data.sh From 7e58c6d1f1161159e60ce2c464af78d10ea3d2a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 15:26:32 +0000 Subject: [PATCH 132/189] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7bfaa4bcf..9e998655f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -24,6 +24,7 @@ jobs: zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 + suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps # Workflow steps steps: @@ -243,7 +244,7 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src + cmake -DCCPP_SUITES=${suites},${suites_ps} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM. run: | From 82dd9ca91dddc246dd5a59556c90fb1a2b0846f5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 11 Mar 2024 17:54:19 +0000 Subject: [PATCH 133/189] Cleanup/Housekeeping. Remove unnecessary interpolation step --- scm/etc/scripts/UFS_IC_generator.py | 41 ++++------------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index af3f45d29..cc570d239 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1917,37 +1917,6 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr pres_adv[t+1,:] = pres_adv[t,:] pres_i_adv[t+1,:] = pres_i_adv[t,:] - if save_comp_data: - # - t_layr = np.zeros([n_files+1,nlevs]) - qv_layr = np.zeros([n_files+1,nlevs]) - u_layr = np.zeros([n_files+1,nlevs]) - v_layr = np.zeros([n_files+1,nlevs]) - p_layr = np.zeros([n_files+1,nlevs]) - - # - for t in range(0,n_files): - from_p[0,:] = stateNATIVE["p_lev"][t,::-1] - to_p[0,:] = stateNATIVE["p_lev"][1,::-1] - log_from_p[0,:] = np.log(from_p[0,:]) - log_to_p[0,:] = np.log(to_p[0,:]) - p_layr[t,:] = stateNATIVE["p_lay"][1,::-1] - for k in range(0,nlevs): dp2[0,k] = to_p[0,k+1] - to_p[0,k] - t_layr[t,:] = fv3_remap.map_scalar(nlevs, log_from_p, stateNATIVE["t_lay"][t:t+1,::-1], \ - dummy, nlevs, log_to_p, 0, 0, 1, np.abs(kord_tm), t_min) - qv_layr[t,:] = fv3_remap.map1_q2(nlevs, from_p, stateNATIVE["qv_lay"][t:t+1,::-1], \ - nlevs, to_p, dp2, 0, 0, 0, kord_tr, q_min) - u_layr[t,:] = fv3_remap.map1_ppm(nlevs, from_p, stateNATIVE["u_lay"][t:t+1,::-1], \ - 0.0, nlevs, to_p, 0, 0, -1, kord_tm) - v_layr[t,:] = fv3_remap.map1_ppm(nlevs, from_p, stateNATIVE["v_lay"][t:t+1,::-1], \ - 0.0, nlevs, to_p, 0, 0, -1, kord_tm) - - t_layr[t+1,:] = t_layr[t,:] - qv_layr[t+1,:] = qv_layr[t,:] - u_layr[t+1,:] = u_layr[t,:] - v_layr[t+1,:] = v_layr[t,:] - p_layr[t+1,:] = p_layr[t,:] - #################################################################################### # # if we had atmf,sfcf files at every timestep (and the SCM timestep is made to match @@ -2116,11 +2085,11 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr if (save_comp_data): comp_data = { "time": stateNATIVE["time"]*sec_in_hr, - "pa" : p_layr[:,::-1], - "ta" : t_layr[:,::-1], - "qv" : qv_layr[:,::-1], - "ua" : u_layr[:,::-1], - "va" : v_layr[:,::-1], + "pa" : stateNATIVE["p_lay"][:,:], + "ta" : stateNATIVE["t_lay"][:,:], + "qv" : stateNATIVE["qv_lay"][:,:], + "ua" : stateNATIVE["u_lay"][:,:], + "va" : stateNATIVE["v_lay"][:,:], "vars2d":vars2d} else: comp_data = {} From 7f7c80b361f18f4536430edfbd3840a1d32fe91f Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 19:54:47 +0000 Subject: [PATCH 134/189] Add capability to use input file with longitude and latitude lists --- scm/etc/scripts/UFS_IC_generator.py | 60 ++++++++++++++----- .../scripts/UFS_forcing_ensemble_generator.py | 54 +++++++++++++---- 2 files changed, 88 insertions(+), 26 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index cc570d239..5734c78f2 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -464,18 +464,20 @@ def get_UFS_IC_data(dir, grid_dir, tile, i, j, old_chgres, lam): #returns dictionaries with the data vgrid_data = get_UFS_vgrid_data(grid_dir) #only needed for ak, bk to calculate pressure - state_data = get_UFS_state_data(vgrid_data, dir, tile, i, j, old_chgres, lam) + (state_data, error_msg) = get_UFS_state_data(vgrid_data, dir, tile, i, j, old_chgres, lam) surface_data = get_UFS_surface_data(dir, tile, i, j, old_chgres, lam) oro_data = get_UFS_oro_data(dir, tile, i, j, lam) - return (state_data, surface_data, oro_data) + return (state_data, surface_data, oro_data, error_msg) ######################################################################################## # ######################################################################################## def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): """Get the state data for the given tile and indices""" - + + state = {} + error_msg=None if lam: nc_file_data = Dataset('{0}/{1}'.format(dir,'gfs_data.nc')) else: @@ -600,7 +602,10 @@ def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): icewat_model_rev[k] = cloud_water - liqwat_model_rev[0,k] (liqwat_model_rev[0,k], dummy_rain, icewat_model_rev[k], dummy_snow) = fv3_remap.mp_auto_conversion(liqwat_model_rev[0,k], icewat_model_rev[k]) - [u_s, u_n, v_w, v_e] = get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam) + [u_s, u_n, v_w, v_e, unknown_grid] = get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam) + if unknown_grid: + error_msg='unknown grid orientation' + return(state,error_msg) #put C/D grid zonal/meridional winds on model pressure levels u_s_model_rev = fv3_remap.mappm(levp_data, pressure_from_data_rev[np.newaxis, :], u_s[np.newaxis, :], nlevs_model, pressure_model_interfaces_rev[np.newaxis, :], 1, 1, -1, 8, ptop_data) @@ -638,7 +643,7 @@ def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): "pa_i": pressure_model_interfaces } - return state + return (state,error_msg) ######################################################################################## # @@ -665,6 +670,10 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam raise Exception(message) nc_file_grid = Dataset('{0}/{1}'.format(dir,filename)) + + # Get grid dimension + nz,nx,ny = np.shape(nc_file_data['u_w']) + if (lam): #strip ghost/halo points and return supergrid @@ -696,7 +705,7 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam east_test_point = np.argmax(test_lon_diff) north_test_point = np.argmax(test_lat_diff) - + unknown=False if east_test_point == 0: #longitude increases most along the positive i axis if north_test_point == 2: @@ -775,7 +784,11 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j,i+1]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j,i+1]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 1: #longitude increases most along the negative i axis if north_test_point == 2: @@ -853,7 +866,11 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j,i-1]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j,i-1]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 2: #longitude increases most along the positive j axis if north_test_point == 0: @@ -929,9 +946,16 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam p3 = fv3_remap.mid_pt_sphere(p1*deg_to_rad, p2*deg_to_rad) e1 = fv3_remap.get_unit_vect2(p1*deg_to_rad, p2*deg_to_rad) (ex, ey) = fv3_remap.get_latlon_vector(p3) - v_e = nc_file_data['u_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ey) + if (j < nx -1): + v_e = nc_file_data['u_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ey) + else: + v_e = nc_file_data['v_w'][:,j,i] else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 3: #longitude increases most along the negative j axis if north_test_point == 0: @@ -1009,12 +1033,15 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j-1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j-1,i]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') - + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True nc_file_grid.close() - return [u_s, u_n, v_w, v_e] + return [u_s, u_n, v_w, v_e, unknown] ######################################################################################## # @@ -2685,8 +2712,11 @@ def main(): # get UFS IC data (TODO: flag to read in RESTART data rather than IC data and implement # different file reads) - (state_data, surface_data, oro_data) = get_UFS_IC_data(in_dir, grid_dir, tile, tile_i,\ - tile_j, old_chgres, lam) + (state_data, surface_data, oro_data, error_msg) = get_UFS_IC_data(in_dir, grid_dir, tile, tile_i,\ + tile_j, old_chgres, lam) + if (error_msg): + print("ERROR: unknown grid orintation") + exit() if not date: # date was not included on command line; look in atmf* file for initial date diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index d057dc6cf..23b90b499 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -20,6 +20,7 @@ parser.add_argument('-latl', '--lat_limits', help='latitude range, separated by a space', nargs=2, type=float, required=False) parser.add_argument('-lons', '--lon_list', help='longitudes, separated by a space', nargs='*', type=float, required=False) parser.add_argument('-lats', '--lat_list', help='latitudes, separated by a space', nargs='*', type=float, required=False) +parser.add_argument('-fxy', '--lonlat_file', help='file containing longitudes and latitude',nargs=1, required=False) parser.add_argument('-nens', '--nensmembers', help='number of SCM UFS ensemble memebers to create', type=int, required=False) parser.add_argument('-dt', '--timestep', help='SCM timestep, in seconds', type=int, default = 3600) parser.add_argument('-cres', '--C_RES', help='UFS spatial resolution', type=int, default = 96) @@ -43,25 +44,29 @@ def main(): if (args.lon_limits and args.lon_list): print("ERROR: Can't provide explicit longitude(s) AND a longitude range") exit() + # end if if (args.lat_limits and args.lat_list): print("ERROR: Can't provide explicit latitude(s) AND a latitude range") exit() + # end if if (args.lon_limits or args.lat_limits) and not args.nensmembers: print("ERROR: Longitude/Latitude range provided, but NOT ensemble count.") exit() - + # end if if (args.nensmembers): npts = args.nensmembers if (args.lat_list or args.lon_list): print("ERROR: Can't provide explicit lon/lat range AND number of points for ensemble generation.") exit() - else: - if (args.lon_list and args.lat_list): - if (len(args.lon_list) == len(args.lat_list)): - npts = len(args.lon_list) - else: - print("ERROR: Number of longitude/latitudes are inconsistent") - exit() + # end if + elif (args.lon_list and args.lat_list): + if (len(args.lon_list) == len(args.lat_list)): + npts = len(args.lon_list) + else: + print("ERROR: Number of longitude/latitudes are inconsistent") + exit() + # end if + # end if ########################################################################### # @@ -83,19 +88,44 @@ def main(): lats[ipt] = args.lat_limits[0] + (args.lat_limits[1]-args.lat_limits[0])*rng1[ipt] else: lats[ipt] = rng1[ipt]*180-90 + # end if if args.lon_limits: lons[ipt] = args.lon_limits[0] + (args.lon_limits[1]-args.lon_limits[0])*rng2[ipt] else: lons[ipt] = rng2[ipt]*360 + # end if + # end for ########################################################################### # - # Use longitude and latitude provided + # Use longitude and latitude provided to command line # ########################################################################### - else: + elif (args.lon_list and args.lat_list): lons = np.asarray(args.lon_list) lats = np.asarray(args.lat_list) - + ########################################################################### + # + # Use longitude and latitude from input file + # + ########################################################################### + elif (args.lonlat_file): + fid = open(args.lonlat_file[0], 'r') + lines = fid.read().split('\n') + lon_list = lines[0] + lons = eval(lon_list) + lat_list = lines[1] + lats = eval(lat_list) + lons = lons[563::] + lats = lats[563::] + npts = len(lons) + else: + print("ERROR: Must provide input points in one of the following formats:") + print(" Using -nens [] -lonl [] -latl [] (e.g. -nens 20 -lonl 30 40 -latl 30 35)") + print(" Using -lons [] -lats [] (e.g. -lons 203 204 205 -lats 30 30 30)") + print(" Using -fxy (e.g. -fxy lonlat.txt w/ -lons [] -lats [])") + exit() + # end if + ########################################################################### # # Create SCM case configuration (etc/case_config) file. @@ -154,6 +184,7 @@ def main(): # count = count + 1 + # end for ########################################################################### # @@ -171,6 +202,7 @@ def main(): #print(' {"case": "' , run["case"] , '", "suite": "' , run["suite"] , '"},') fileID.write(' {"case": "' + run["case"] + '", "suite": "' + run["suite"] + '"},') fileID.write('\n') + # end for fileID.write(' ]') fileID.close() From 650e207aa244b1c46b4e7c1e7fc2ccb8a44e4733 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 20:02:35 +0000 Subject: [PATCH 135/189] Omission from previous commit --- scm/etc/scripts/UFS_forcing_ensemble_generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index 23b90b499..cb7f8e426 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -115,8 +115,6 @@ def main(): lons = eval(lon_list) lat_list = lines[1] lats = eval(lat_list) - lons = lons[563::] - lats = lats[563::] npts = len(lons) else: print("ERROR: Must provide input points in one of the following formats:") From dac377e3628b3632d27d3574718fee8fc56ac17a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 20:42:19 +0000 Subject: [PATCH 136/189] Added utility script to create input file containing replay points --- scm/etc/scripts/create_lonlat4replay.py | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 scm/etc/scripts/create_lonlat4replay.py diff --git a/scm/etc/scripts/create_lonlat4replay.py b/scm/etc/scripts/create_lonlat4replay.py new file mode 100755 index 000000000..844095fcc --- /dev/null +++ b/scm/etc/scripts/create_lonlat4replay.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +################################################################################################# +# Dependencies +################################################################################################# +import os +import numpy as np +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-d', '--dir', help='path to UFS Regression Test output', required=True) +parser.add_argument('-n', '--case_name', help='name of case', required=True) +parser.add_argument('-lonl', '--lon_range', help='longitude range, separated by a space', nargs=2, type=int, required=True) +parser.add_argument('-latl', '--lat_range', help='latitude range, separated by a space', nargs=2, type=int, required=True) +parser.add_argument('-dlat', '--dlat', help='latitude spacing', type=int, required=True) +parser.add_argument('-dlon', '--dlon', help='longitude spacing', type=int, required=True) +parser.add_argument('-dt', '--timestep', help='SCM timestep, in seconds', type=int, required=True) +parser.add_argument('-cres', '--C_RES', help='UFS spatial resolution', type=int, required=True) +parser.add_argument('-sdf', '--suite', help='CCPP suite definition file to use for ensemble', required=True) +parser.add_argument('-fout', '--fileOUT', help='Text file containing lon/lat points', required=False, default='lonlats.txt') + +def main(): + + # Get command line arguments + args = parser.parse_args() + dir = args.dir+'/'+args.case_name + + com_pre = './UFS_forcing_ensemble_generator.py -d '+dir+' -sc --C_RES '+str(args.C_RES)+' -dt '+str(args.timestep)+' -n '+args.case_name+' -sdf '+args.suite + + lons = [] + lats = [] + count = 0 + for lon in range(args.lon_range[0],args.lon_range[1],args.dlon): + for lat in range(args.lat_range[0],args.lat_range[1],args.dlat): + count = count + 1 + lons.append(lon) + lats.append(lat) + # end for + # end for + + comA = '-lons ' + for ij in lons: + comA = comA + str(ij)+' ' + # end for + comB = '-lats ' + for ij in lats: + comB = comB + str(ij)+' ' + # end for + com = com_pre+comA+comB + print(com) + print("#######################################################################################") + + # For python list input file + comA = '' + countA = 0 + for ij in lons: + countA = countA + 1 + if (countA < count): + comA = comA + str(ij)+', ' + else: + comA = comA + str(ij) + #end if + # end for + + comB = '' + countB = 0 + for ij in lats: + countB = countB + 1 + if (countB < count): + comB = comB + str(ij)+', ' + else: + comB = comB + str(ij) + # end if + # end for + + fileID = open(args.fileOUT,'w') + fileID.write(comA+"\n") + fileID.write(comB+"\n") + fileID.close() + com = com_pre+' -fxy '+args.fileOUT + print(com) + +if __name__ == '__main__': + main() From b93bc5110df919812dcd9aa20d9260c316811974 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 21:04:23 +0000 Subject: [PATCH 137/189] Added step to skip gridpoint in ensemble if error during case generation --- .../scripts/UFS_forcing_ensemble_generator.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index cb7f8e426..c4a533127 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -153,35 +153,37 @@ def main(): print(com) os.system(com) - # Add case to ensemble list. - case_list = case_list + '"'+case_name+'"' - if (count != npts-1): case_list = case_list + ', ' + if (os.path.isfile(file_scminput)): + # Add case to ensemble list. + case_list = case_list + '"'+case_name+'"' + if (count != npts-1): case_list = case_list + ', ' - # What is the surface type? (get from SCM input file) - dataset = xr.open_dataset(file_scminput) - sfc_type = int(np.round_(dataset.slmsk.values[0])) + # What is the surface type? (get from SCM input file) + dataset = xr.open_dataset(file_scminput) + sfc_type = int(np.round_(dataset.slmsk.values[0])) - # Create case_config file(s) - fileOUT = "../../etc/case_config/"+case_name+".nml" - fileID = open(fileOUT, 'w') - fileID.write('$case_config') - fileID.write('\n') - fileID.write('case_name = ' + "'" + case_name + "',") - fileID.write('\n') - fileID.write('sfc_type = ' + str(sfc_type) + ",") - fileID.write('\n') - for opts in case_config: - fileID.write(opts["name"] + ' = ' + opts["values"] + ",") + # Create case_config file(s) + fileOUT = "../../etc/case_config/"+case_name+".nml" + fileID = open(fileOUT, 'w') + fileID.write('$case_config') fileID.write('\n') - fileID.write('$end') - fileID.write('\n') - fileID.close() + fileID.write('case_name = ' + "'" + case_name + "',") + fileID.write('\n') + fileID.write('sfc_type = ' + str(sfc_type) + ",") + fileID.write('\n') + for opts in case_config: + fileID.write(opts["name"] + ' = ' + opts["values"] + ",") + fileID.write('\n') + fileID.write('$end') + fileID.write('\n') + fileID.close() - # Add case to dictionary to be used by run_scm.py - run_list.append({"case": case_name, "suite": args.suite}) + # Add case to dictionary to be used by run_scm.py + run_list.append({"case": case_name, "suite": args.suite}) - # - count = count + 1 + # + count = count + 1 + # end if # end for ########################################################################### From 46ecc1882b5247c37abaccdef9e5e4fe332758e9 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 28 Mar 2024 15:26:05 -0400 Subject: [PATCH 138/189] update CCPP_typedefs to work with PR#181 --- ccpp/physics | 2 +- scm/src/CCPP_typedefs.F90 | 2 +- scm/src/CCPP_typedefs.meta | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccpp/physics b/ccpp/physics index 6b0599ef6..4d497be1c 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 6b0599ef6c7ea076336b1f073899c5b97e37d584 +Subproject commit 4d497be1c5c5dddee9afcf8662ac30e005062b8e diff --git a/scm/src/CCPP_typedefs.F90 b/scm/src/CCPP_typedefs.F90 index 878422d39..0b49ed010 100644 --- a/scm/src/CCPP_typedefs.F90 +++ b/scm/src/CCPP_typedefs.F90 @@ -622,7 +622,7 @@ subroutine gfs_interstitial_create (Interstitial, IM, Model) allocate (Interstitial%sigma (IM)) allocate (Interstitial%sigmaf (IM)) allocate (Interstitial%sigmafrac (IM,Model%levs)) - allocate (Interstitial%sigmatot (IM,Model%levs)) + allocate (Interstitial%sigmatot (IM,Model%levs+1)) allocate (Interstitial%snowc (IM)) allocate (Interstitial%snohf (IM)) allocate (Interstitial%snowmt (IM)) diff --git a/scm/src/CCPP_typedefs.meta b/scm/src/CCPP_typedefs.meta index 8ab9f4ba2..8208bd1d5 100644 --- a/scm/src/CCPP_typedefs.meta +++ b/scm/src/CCPP_typedefs.meta @@ -1952,7 +1952,7 @@ standard_name = convective_updraft_area_fraction_at_model_interfaces long_name = convective updraft area fraction at model interfaces units = frac - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_loop_extent,vertical_interface_dimension) type = real kind = kind_phys [skip_macro] From efa103fc05fe9f42dbad6aea8d809be127119028 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 28 Mar 2024 15:37:37 -0400 Subject: [PATCH 139/189] update ccpp/framework --- ccpp/framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/framework b/ccpp/framework index 219f2e9c8..f0b9a18b0 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit 219f2e9c88b7b774becac2bd1453696e105af1c4 +Subproject commit f0b9a18b005d950cb9b0038fbc827b6b37500f43 From adf7b681a9dbb51e5a54697b9105ea06cf2b4652 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 29 Mar 2024 13:53:18 -0400 Subject: [PATCH 140/189] update ccpp/physics submodule --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 4d497be1c..212f3ff2f 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 4d497be1c5c5dddee9afcf8662ac30e005062b8e +Subproject commit 212f3ff2f909a43597ce4c9063b09b4e1f2b3e3d From 9e3663a10bbce810f37197e3ad67d3520f1d9852 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 3 Apr 2024 17:32:07 +0000 Subject: [PATCH 141/189] Synced physics --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index aabe63844..69917a6e2 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit aabe63844ae4a81cc20d1937d737c078380a0392 +Subproject commit 69917a6e274f8d4f4b264b2dee58b721088a9918 From 46669778b89378617cab58094aebfb9575722066 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 10 Apr 2024 13:30:32 -0400 Subject: [PATCH 142/189] update ccpp/physics submodule pointer --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 212f3ff2f..fd85f43c4 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 212f3ff2f909a43597ce4c9063b09b4e1f2b3e3d +Subproject commit fd85f43c41e823f969f61aac9bde611836ebac3a From c25071b2f6e1f4b331f16a9cc32d5c20c529a01b Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 10 Apr 2024 19:20:43 +0000 Subject: [PATCH 143/189] Address reviewers comments --- scm/etc/scripts/UFS_IC_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 5734c78f2..cac0d2773 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1202,7 +1202,7 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): # fractional grid tiice_in = read_NetCDF_surface_var(nc_file, 'tiice', i, j, old_chgres, missing_variable_ice_layers) - # soil color + # soil color (From the UFS FV3: io/fv3atm_sfc_io.F90) if (slmsk_in == 1): scolor_in = 4 else: @@ -2715,7 +2715,8 @@ def main(): (state_data, surface_data, oro_data, error_msg) = get_UFS_IC_data(in_dir, grid_dir, tile, tile_i,\ tile_j, old_chgres, lam) if (error_msg): - print("ERROR: unknown grid orintation") + print(error_msg) + print("STOPPING") exit() if not date: From 88b3df8ad61c5863ced06ce99dff275d851935a7 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 10 Apr 2024 19:22:10 +0000 Subject: [PATCH 144/189] Update physics --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 69917a6e2..5e56dcef4 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 69917a6e274f8d4f4b264b2dee58b721088a9918 +Subproject commit 5e56dcef4175c0f3ff0b98ce4f04765f55c7fdcf From 51c54202f20bab8ce989622154997381f1f7dd2f Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 10 Apr 2024 19:40:19 +0000 Subject: [PATCH 145/189] Revert submodules --- .gitmodules | 4 ++-- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1421ef8f..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/dustinswales/ccpp-physics - branch = replay_fixLSMinit + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 5e56dcef4..6d8fccbea 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 5e56dcef4175c0f3ff0b98ce4f04765f55c7fdcf +Subproject commit 6d8fccbea12c11387c0bc457dcb3855574cd29aa From 472b7d5792a53d0361c9d0940cb3b34543b417b3 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 11 Apr 2024 13:51:28 -0400 Subject: [PATCH 146/189] changes from @bluefinweiwei for PR#419 --- scm/etc/scripts/UFS_IC_generator.py | 8 +- scm/etc/scripts/extract_FV3GFS_column_ic.py | 415 -------------------- 2 files changed, 4 insertions(+), 419 deletions(-) delete mode 100755 scm/etc/scripts/extract_FV3GFS_column_ic.py diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index cac0d2773..5e290d3df 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1351,7 +1351,7 @@ def get_UFS_oro_data(dir, tile, i, j, lam): if lam: nc_file = Dataset('{0}/{1}'.format(dir,'oro_data.nc')) else: - filename_pattern = 'oro_data.tile{0}.nc'.format(tile) + filename_pattern = 'oro*.tile{0}.nc'.format(tile) for f_name in os.listdir(dir): if fnmatch.fnmatch(f_name, filename_pattern): filename = f_name @@ -1500,8 +1500,8 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr atm_ftag = 'atmf*.tile{0}.nc'.format(tile) sfc_ftag = 'sfcf*.tile{0}.nc'.format(tile) else: - atm_ftag = 'atmf*.nc' - sfc_ftag = 'sfcf*.nc' + atm_ftag = '*atmf*.nc' + sfc_ftag = '*sfcf*.nc' # Get list of UFS history files with 3D ATMospheric state variables. atm_filenames = [] @@ -2646,7 +2646,7 @@ def write_comparison_file(comp_data, case_name, date, surface): ######################################################################################## def find_date(forcing_dir): - atm_ftag = 'atmf*.nc' + atm_ftag = '*atmf*.nc' atm_filenames = [] for f_name in os.listdir(forcing_dir): diff --git a/scm/etc/scripts/extract_FV3GFS_column_ic.py b/scm/etc/scripts/extract_FV3GFS_column_ic.py deleted file mode 100755 index 6684125c1..000000000 --- a/scm/etc/scripts/extract_FV3GFS_column_ic.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python -from netCDF4 import Dataset -import numpy as np - -# define date -YYYY=2014 -MM=8 -DD=1 -HH=0 -MI=0 -SC=0 - -# define path to FV3 GFS initial and boundary conditions -icpath='../../data/raw_case_input/UFS_test_ics' -fixpath='../../data/raw_case_input/UFS_test_ics' -print (icpath) -# defint i,j and tile to extract colmn -ipt=16 -jpt=41 -tilenum=2 - -ipt2=ipt*2+1 -jpt2=jpt*2+1 - -# I have surface cycle on the sfc_data initial conditions to get the proper surface fields -infile1='%s/gfs_data.tile%i.nc' % (icpath,tilenum) -infile2='%s/sfc_data.tile%i.nc' %(icpath,tilenum) -infile3='%s/C96_oro_data.tile%i.nc' %(fixpath,tilenum) -infile4='%s/C96_grid.tile%i.nc' %(fixpath,tilenum) -infile5='%s/gfs_ctrl.nc' %icpath -ncin1=Dataset(infile1) -ncin2=Dataset(infile2) -ncin3=Dataset(infile3) -ncin4=Dataset(infile4) -ncin5=Dataset(infile5) -# assume model contains one less level than the cold start spectral GFS initial conditions -nlevs=len(ncin1.dimensions['lev'])-1 -# pick off lat and lon from i and j point defined above -lon0=ncin4['x'][jpt2,ipt2] -lat0=ncin4['y'][jpt2,ipt2] - -# extract out area of grid cell -area_in=ncin4['area'][jpt2-1:jpt2+1,ipt2-1:ipt2+1] -print (lat0,lon0) - -# upper air fields from initial conditions -zh=ncin1['zh'][::-1,jpt,ipt] -uw1=ncin1['u_w'][::-1,jpt,ipt] -uw2=ncin1['u_w'][::-1,jpt,ipt+1] -us1=ncin1['u_s'][::-1,jpt,ipt] -us2=ncin1['u_s'][::-1,jpt+1,ipt] -vw1=ncin1['v_w'][::-1,jpt,ipt] -vw2=ncin1['v_w'][::-1,jpt,ipt+1] -vs1=ncin1['v_s'][::-1,jpt,ipt] -vs2=ncin1['v_s'][::-1,jpt+1,ipt] -ucomp=0.25*(uw1+uw2+us1+us2) # estimate u winds on the a grid -vcomp=0.25*(vw1+vw2+vs1+vs2) # estimate v winds on the a grid -sphum=ncin1['sphum'][::-1,jpt,ipt] -# o3 and qv are taken from ics. -o3=ncin1['o3mr'][::-1,jpt,ipt] -liqwat=ncin1['liq_wat'][:-1,jpt,ipt] - -# surface pressure and skin temperature -ps=ncin1['ps'][jpt,ipt] -ts=ncin2['tsea'][jpt,ipt] - -# land state -stc_in=ncin2['stc'][:,jpt,ipt] -smc_in=ncin2['smc'][:,jpt,ipt] -slc_in=ncin2['slc'][:,jpt,ipt] -tg3_in=ncin2['tg3'][jpt,ipt] - -# surface properties -uustar_in=ncin2['uustar'][jpt,ipt] -alvsf=ncin2['alvsf'][jpt,ipt] -alvwf=ncin2['alvwf'][jpt,ipt] -alnsf=ncin2['alnsf'][jpt,ipt] -alnwf=ncin2['alnwf'][jpt,ipt] -facsf_in=ncin2['facsf'][jpt,ipt] -facwf_in=ncin2['facwf'][jpt,ipt] -styp_in=ncin2['stype'][jpt,ipt] -slope_in=ncin2['slope'][jpt,ipt] -vtyp_in=ncin2['vtype'][jpt,ipt] -vfrac_in=ncin2['vfrac'][jpt,ipt] -shdmin_in=ncin2['shdmin'][jpt,ipt] -shdmax_in=ncin2['shdmax'][jpt,ipt] -zorl_in=ncin2['zorl'][jpt,ipt] -slmsk_in=ncin2['slmsk'][jpt,ipt] -canopy_in=ncin2['canopy'][jpt,ipt] -hice_in=ncin2['hice'][jpt,ipt] -fice_in=ncin2['fice'][jpt,ipt] -tisfc_in=ncin2['tisfc'][jpt,ipt] -snwdph_in=ncin2['snwdph'][jpt,ipt] -snoalb_in=ncin2['snoalb'][jpt,ipt] - -# orographyic properties -stddev_in=ncin3['stddev'][jpt,ipt] -convexity_in=ncin3['convexity'][jpt,ipt] -oa1_in=ncin3['oa1'][jpt,ipt] -oa2_in=ncin3['oa2'][jpt,ipt] -oa3_in=ncin3['oa3'][jpt,ipt] -oa4_in=ncin3['oa4'][jpt,ipt] -ol1_in=ncin3['ol1'][jpt,ipt] -ol2_in=ncin3['ol2'][jpt,ipt] -ol3_in=ncin3['ol3'][jpt,ipt] -ol4_in=ncin3['ol4'][jpt,ipt] -theta_in=ncin3['theta'][jpt,ipt] -gamma_in=ncin3['gamma'][jpt,ipt] -sigma_in=ncin3['sigma'][jpt,ipt] -elvmax_in=ncin3['elvmax'][jpt,ipt] - -# vertical coordinate definition -ak=ncin5['vcoord'][0,::-1] -bk=ncin5['vcoord'][1,::-1] - -#calculate temperature -rdgas = 287.05 -rvgas = 461.50 -zvir = rvgas/rdgas - 1. -grav=9.80665 -gz=zh*grav -pn1=np.zeros([nlevs+1]) -temp=np.zeros([nlevs]) -for k in range(nlevs+1): - pn1[k]=np.log(ak[k]+ps*bk[k]) -for k in range(nlevs): - temp[k] = (gz[k]-gz[k+1])/( rdgas*(pn1[k+1]-pn1[k])*(1.+zvir*sphum[k]) ) - -# open output file -nc = Dataset('../../data/processed_case_input/fv3_model_point.nc', mode='w') -nc.description = "FV3GFS model profile input (no forcing)" - -time = nc.createDimension('time',None) -levels = nc.createDimension('levels',None) -nsoil = nc.createDimension('nsoil',None) -t = nc.createVariable('time',np.float64,('time',)) -t.units = "s" -t.description = "elapsed time since the beginning of the simulation" -z = nc.createVariable('levels',np.float64,('levels',)) -z.units = "Pa" -z.description = "pressure levels" -#scalars -iyr = nc.createVariable('scalars/init_year',np.int32) -imo = nc.createVariable('scalars/init_month',np.int32) -idy = nc.createVariable('scalars/init_day',np.int32) -ihr = nc.createVariable('scalars/init_hour',np.int32) -imi = nc.createVariable('scalars/init_minute',np.int32) -isc = nc.createVariable('scalars/init_second',np.int32) -ivegsrc = nc.createVariable('scalars/vegsrc',np.int32) -ivegtyp = nc.createVariable('scalars/vegtyp',np.int32) -isoiltyp = nc.createVariable('scalars/soiltyp',np.int32) -islopetyp = nc.createVariable('scalars/slopetyp',np.int32) -ivegfrac = nc.createVariable('scalars/vegfrac',np.float) -ishdmin = nc.createVariable('scalars/shdmin',np.float) -ishdmax = nc.createVariable('scalars/shdmax',np.float) -izorl = nc.createVariable('scalars/zorl',np.float) -islmsk = nc.createVariable('scalars/slmsk',np.float) -icanopy = nc.createVariable('scalars/canopy',np.float) -ihice = nc.createVariable('scalars/hice',np.float) -ifice = nc.createVariable('scalars/fice',np.float) -itisfc = nc.createVariable('scalars/tisfc',np.float) -isnwdph = nc.createVariable('scalars/snwdph',np.float) -isnoalb = nc.createVariable('scalars/snoalb',np.float) -isncovr = nc.createVariable('scalars/sncovr',np.float) -itg3 = nc.createVariable('scalars/tg3',np.float) -iuustar = nc.createVariable('scalars/uustar',np.float) - -iyr.units = "years" -iyr.description = "year at time of initial values" -imo.units = "months" -imo.description = "month at time of initial values" -idy.units = "days" -idy.description = "day at time of initial values" -ihr.units = "hours" -ihr.description = "hour at time of initial values" -imi.units = "minutes" -imi.description = "minute at time of initial values" -isc.units = "seconds" -isc.description = "second at time of initial values" -ivegsrc.description = "vegetation soure (1-2)" -ivegtyp.description = "vegetation type (1-12)" -isoiltyp.description = "soil type (1-12)" -islopetyp.description = "slope type (1-9)" -ivegfrac.description = "vegetation fraction" -ishdmin.description = "minimum vegetation fraction" -ishdmax.description = "maximum vegetation fraction" -izorl.description = "surface roughness length" -islmsk.description = "land-sea-ice mask" -icanopy.description = "canopy moisture" -ihice.description = "ice thickness" -ifice.description = "ice fraction" -itisfc.description = "ice temperature" -isnwdph.description = "snow depth" -isnoalb.description = "snow albedo" -isncovr.description = "snow cover" -itg3.description = "deep soil temperature" -itg3.units = "K" -iuustar.description = "frication velocity" -iuustar.units = "m2s-2?" - - -#initial -ic_t = nc.createVariable('initial/temp',np.float64,('levels',)) -ic_qt = nc.createVariable('initial/qt',np.float64,('levels',)) -ic_ql = nc.createVariable('initial/ql',np.float64,('levels',)) -ic_qi = nc.createVariable('initial/qi',np.float64,('levels',)) -ic_u = nc.createVariable('initial/u',np.float64,('levels',)) -ic_v = nc.createVariable('initial/v',np.float64,('levels',)) -ic_tke = nc.createVariable('initial/tke',np.float64,('levels',)) -ic_o3 = nc.createVariable('initial/ozone',np.float64,('levels',)) -ic_stc = nc.createVariable('initial/stc',np.float64,('nsoil',)) -ic_smc = nc.createVariable('initial/smc',np.float64,('nsoil',)) -ic_slc = nc.createVariable('initial/slc',np.float64,('nsoil',)) -ic_t.units = "K" -#ic_t.description = "initial profile of ice-liquid water potential temperature" -ic_t.description = "initial profile of temperature" -ic_qt.units = "kg kg^-1" -ic_qt.description = "initial profile of total water specific humidity" -ic_ql.units = "kg kg^-1" -ic_ql.description = "initial profile of liquid water specific humidity" -ic_qi.units = "kg kg^-1" -ic_qi.description = "initial profile of ice water specific humidity" -ic_u.units = "m s^-1" -ic_u.description = "initial profile of E-W horizontal wind" -ic_v.units = "m s^-1" -ic_v.description = "initial profile of N-S horizontal wind" -ic_tke.units = "m^2 s^-2" -ic_tke.description = "initial profile of turbulence kinetic energy" -ic_o3.units = "kg kg^-1" -ic_o3.description = "initial profile of ozone mass mixing ratio" -ic_stc.units = "K" -ic_stc.description = "initial profile of soil temperature" -ic_smc.units = "kg" -ic_smc.description = "initial profile of soil moisture" -ic_slc.units = "kg" -ic_slc.description = "initial profile of soil liquid moisture" - -lat=nc.createVariable('forcing/lat',np.float64,('time',)) -lat.units = "degrees N" -lat.description = "latitude of column" -lon=nc.createVariable('forcing/lon',np.float64,('time',)) -lon.units = "degrees E" -lon.description = "longitude of column" -p_surf=nc.createVariable('forcing/p_surf',np.float64,('time',)) -p_surf.units = "Pa" -p_surf.description = "surface pressure" -T_surf=nc.createVariable('forcing/T_surf',np.float64,('time',)) -T_surf.units = "K" -T_surf.description = "surface absolute temperature" -area1=nc.createVariable('scalars/area',np.float64,('time',)) -alb1=nc.createVariable('scalars/alvsf',np.float64,('time',)) -alb2=nc.createVariable('scalars/alnsf',np.float64,('time',)) -alb3=nc.createVariable('scalars/alvwf',np.float64,('time',)) -alb4=nc.createVariable('scalars/alnwf',np.float64,('time',)) -stddev=nc.createVariable('scalars/stddev',np.float64,('time',)) -convexity=nc.createVariable('scalars/convexity',np.float64,('time',)) -oa1=nc.createVariable('scalars/oa1',np.float64,('time',)) -oa2=nc.createVariable('scalars/oa2',np.float64,('time',)) -oa3=nc.createVariable('scalars/oa3',np.float64,('time',)) -oa4=nc.createVariable('scalars/oa4',np.float64,('time',)) -ol1=nc.createVariable('scalars/ol1',np.float64,('time',)) -ol2=nc.createVariable('scalars/ol2',np.float64,('time',)) -ol3=nc.createVariable('scalars/ol3',np.float64,('time',)) -ol4=nc.createVariable('scalars/ol4',np.float64,('time',)) -theta=nc.createVariable('scalars/theta',np.float64,('time',)) -gamma=nc.createVariable('scalars/gamma',np.float64,('time',)) -sigma=nc.createVariable('scalars/sigma',np.float64,('time',)) -elvmax=nc.createVariable('scalars/elvmax',np.float64,('time',)) -facsf=nc.createVariable('scalars/facsf',np.float64,('time',)) -facwf=nc.createVariable('scalars/facwf',np.float64,('time',)) -area1.units = "m^2" -alb1.units = "None" -alb2.units = "None" -alb3.units = "None" -alb4.units = "None" -facsf.units = "None" -facwf.units = "None" -area1.description = "grid cell area" -alb1.description = "uv+visible black sky albedo (z=60 degree)" -alb2.description = "near IR black sky albedo (z=60 degree)" -alb3.description = "uv+visible white sky albedo" -alb4.description = "near IR white sky albedo" -stddev.description = "surface orography standard deviation" -facsf.description = "fraction of grid cell with strong sun angle albedo dependence" -facwf.description = "fraction of grid cell with weak sun angle albedo dependence" -w_ls=nc.createVariable('forcing/w_ls',np.float64,('levels','time',)) -w_ls.units = "m s^-1" -w_ls.description = "large scale vertical velocity" -omega=nc.createVariable('forcing/omega',np.float64,('levels','time',)) -omega.units = "Pa s^-1" -omega.description = "large scale pressure vertical velocity" -u_g=nc.createVariable('forcing/u_g',np.float64,('levels','time',)) -u_g.units = "m s^-1" -u_g.description = "large scale geostrophic E-W wind" -v_g=nc.createVariable('forcing/v_g',np.float64,('levels','time',)) -v_g.units = "m s^-1" -v_g.description = "large scale geostrophic N-S wind" -u_nudge=nc.createVariable('forcing/u_nudge',np.float64,('levels','time',)) -u_nudge.units = "m s^-1" -u_nudge.description = "E-W wind to nudge toward" -v_nudge=nc.createVariable('forcing/v_nudge',np.float64,('levels','time',)) -v_nudge.units = "m s^-1" -v_nudge.description = "N-S wind to nudge toward" -T_nudge=nc.createVariable('forcing/T_nudge',np.float64,('levels','time',)) -T_nudge.units = "K" -T_nudge.description = "absolute temperature to nudge toward" -thil_nudge=nc.createVariable('forcing/thil_nudge',np.float64,('levels','time',)) -thil_nudge.units = "K" -thil_nudge.description = "potential temperature to nudge toward" -qt_nudge=nc.createVariable('forcing/qt_nudge',np.float64,('levels','time',)) -qt_nudge.units = "kg kg^-1" -qt_nudge.description = "q_t to nudge toward" -dT_dt_rad=nc.createVariable('forcing/dT_dt_rad',np.float64,('levels','time',)) -dT_dt_rad.units = "K s^-1" -dT_dt_rad.description = "prescribed radiative heating rate" -h_advec_thetail=nc.createVariable('forcing/h_advec_thetail',np.float64,('levels','time',)) -h_advec_thetail.units = "K s^-1" -h_advec_thetail.description = "prescribed theta_il tendency due to horizontal advection" -v_advec_thetail=nc.createVariable('forcing/v_advec_thetail',np.float64,('levels','time',)) -v_advec_thetail.units = "K s^-1" -v_advec_thetail.description = "prescribed theta_il tendency due to vertical advection" -h_advec_qt=nc.createVariable('forcing/h_advec_qt',np.float64,('levels','time',)) -h_advec_qt.units = "kg kg^-1 s^-1" -h_advec_qt.description = "prescribed q_t tendency due to horizontal advection" -v_advec_qt=nc.createVariable('forcing/v_advec_qt',np.float64,('levels','time',)) -v_advec_qt.units = "kg kg^-1 s^-1" -v_advec_qt.description = "prescribed q_t tendency due to vertical advection" - - -# date -iyr[:]=YYYY -imo[:]=MM -idy[:]=DD -ihr[:]=HH -imi[:]=MI -isc[:]=SC -# axes -t[0]=0 - -#ics -ic_t[:] = temp[0:nlevs] -ic_qt[:] = sphum[0:nlevs] -ic_ql[:] = liqwat[0:nlevs] -ic_qi[:] = 0.0 -ic_u[:] = ucomp[0:nlevs] -ic_v[:] = vcomp[0:nlevs] -ic_tke[:] = 0.0 -ic_o3[:] = o3[0:nlevs] -ic_stc[:] = stc_in -ic_smc[:] = smc_in -ic_slc[:] = slc_in - -lat[:]=lat0 -lon[:]=lon0 - -p_surf[:]=ps -T_surf[:]=ts -area1[:]=area_in.sum() -alb1[:]=alvsf -alb2[:]=alnsf -alb3[:]=alvwf -alb4[:]=alnwf -stddev[:]=stddev_in -convexity[:]=convexity_in -oa1[:]=oa1_in -oa2[:]=oa2_in -oa3[:]=oa3_in -oa4[:]=oa4_in -ol1[:]=ol1_in -ol2[:]=ol2_in -ol3[:]=ol3_in -ol4[:]=ol4_in -theta[:]=theta_in -gamma[:]=gamma_in -sigma[:]=sigma_in -elvmax[:]=elvmax_in -facsf[:]=facsf_in -facwf[:]=facwf_in -z[:]=np.exp(pn1[0:nlevs]) -w_ls[:]=0.0 -omega[:]=0.0 -u_g[:]=0.0 -v_g[:]=0.0 -u_nudge[:]=0.0 -v_nudge[:]=0.0 -T_nudge[:]=0.0 -thil_nudge[:]=0.0 -qt_nudge[:]=0.0 -dT_dt_rad[:]=0.0 -h_advec_thetail[:]=0.0 -v_advec_thetail[:]=0.0 -h_advec_qt[:]=0.0 -v_advec_qt[:]=0.0 - -ivegsrc[:] = 1 -ivegtyp[:] = vtyp_in -isoiltyp[:] = styp_in -islopetyp[:] = slope_in -ishdmin[:] = shdmin_in -ishdmax[:] = shdmax_in -izorl[:] = zorl_in -islmsk[:] = slmsk_in -icanopy[:] = canopy_in -ihice[:] = hice_in -ifice[:] = fice_in -itisfc[:] = tisfc_in -isnwdph[:] = snwdph_in -isnoalb[:] = snoalb_in -isncovr[:] = 0.0 -itg3[:] = tg3_in -iuustar[:] = uustar_in -ivegfrac[:]=vfrac_in - -nc.close() - From 048d34588e80a94e8e023e638a889b44437b00bb Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 11 Apr 2024 16:04:50 -0400 Subject: [PATCH 147/189] add HR3 suite and physics namelist from #445 from @bluefinweiwei --- ccpp/physics_namelists/input_GFS_v17_H3.nml | 171 ++++++++++++++++++++ ccpp/suites/suite_SCM_GFS_v17_HR3.xml | 88 ++++++++++ ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml | 69 ++++++++ scm/src/suite_info.py | 1 + 4 files changed, 329 insertions(+) create mode 100644 ccpp/physics_namelists/input_GFS_v17_H3.nml create mode 100644 ccpp/suites/suite_SCM_GFS_v17_HR3.xml create mode 100644 ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml diff --git a/ccpp/physics_namelists/input_GFS_v17_H3.nml b/ccpp/physics_namelists/input_GFS_v17_H3.nml new file mode 100644 index 000000000..10d60a77e --- /dev/null +++ b/ccpp/physics_namelists/input_GFS_v17_H3.nml @@ -0,0 +1,171 @@ +&gfs_physics_nml + fhzero = 6 + h2o_phys = .true. + ldiag3d = .true. + qdiag3d = .true. + print_diff_pgr = .false. + fhcyc = 24 + use_ufo = .true. + pre_rad = .false. + imp_physics = 8 + iovr = 3 + ltaerosol = .false. + lradar = .true. + ttendlim = -999 + dt_inner = 225.0 + sedi_semi = .true. + decfl = 10 + oz_phys = .false. + oz_phys_2015 = .true. + lsoil_lsm = 4 + do_mynnedmf = .false. + do_mynnsfclay = .false. + icloud_bl = 1 + bl_mynn_edmf = 1 + bl_mynn_tkeadvect = .true. + bl_mynn_edmf_mom = 1 + do_ugwp = .false. + do_tofd = .false. + gwd_opt = 2 + do_ugwp_v0 = .false. + do_ugwp_v1 = .true. + do_ugwp_v0_orog_only = .false. + do_ugwp_v0_nst_only = .false. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_ugwp_v1_orog_only = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false. + pdfcld = .false. + fhswr = 1200. + fhlwr = 1200. + ialb = 1 + iems = 1 + iaer = 5111 + icliq_sw = 2 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + shal_cnv = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + satmedmf = .true. + isatmedmf = 1 + lheatstrg = .true. + lseaspray = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + imfshalcnv = 2 + imfdeepcnv = 2 + ras = .false. + cdmbgwd = 10.0,3.5,1.0,1.0 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = 2 + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = 3 + iopt_trs = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + debug = .false. + nstf_name = 2,1,0,0,0 + nst_anl = .true. + psautco = 0.0008,0.0005 + prautco = 0.00015,0.00015 + lgfdlmprad = .false. + effr_in = .true. + ldiag_ugwp = .false. + fscav_aero = "'*:0.0'" + do_sppt = .false. + do_shum = .false. + do_skeb = .false. + do_RRTMGP = .false. + doGP_cldoptics_LUT = .true. + doGP_lwscat = .true. + active_gases = 'h2o_co2_o3_n2o_ch4_o2' + ngases = 6 + rrtmgp_root = '../../ccpp/physics/physics/rte-rrtmgp/' + lw_file_gas = 'rrtmgp/data/rrtmgp-data-lw-g128-210809.nc' + lw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-lw.nc' + sw_file_gas = 'rrtmgp/data/rrtmgp-data-sw-g112-210809.nc' + sw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-sw.nc' + rrtmgp_nGptsSW = 112 + rrtmgp_nGptsLW = 128 + rrtmgp_nBandsLW = 16 + rrtmgp_nBandsSW = 14 + frac_grid = .true. + cplchm = .false. + cplflx = .false. + cplice = .false. + cplwav = .false. + cplwav2atm = .false. + do_ca = .false. + ca_global = .false. + ca_sgs = .false. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + nthresh = 18 + ca_trigger = .true. + nspinup = 1 + iseed_ca = 12345 + lndp_type = 0 + n_var_lndp = 0 + do_ccpp_suite_sim = .false. +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_version = 1 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 2 + knob_ugwp_ndx4lh = 4 + knob_ugwp_palaunch = 275.0e2 + knob_ugwp_nslope = 0 + knob_ugwp_lzmax = 15.750e3 + knob_ugwp_lzmin = 0.75e3 + knob_ugwp_lzstar = 2.0e3 + knob_ugwp_taumin = 0.25e-3 + knob_ugwp_tauamp = 1.5e-3 + knob_ugwp_lhmet = 200.0e3 + knob_ugwp_orosolv = 'pss-1986' +/ + +&ccpp_suite_sim_nml + suite_sim_file = '' + nprc_sim = 7 + prc_LWRAD_cfg = 0, 0, 1 + prc_SWRAD_cfg = 0, 0, 2 + prc_PBL_cfg = 1, 0, 3 + prc_GWD_cfg = 1, 0, 4 + prc_SCNV_cfg = 1, 1, 5 + prc_DCNV_cfg = 1, 1, 6 + prc_cldMP_cfg = 1, 1, 7 +/ diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3.xml new file mode 100644 index 000000000..92effe13c --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3.xml @@ -0,0 +1,88 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml new file mode 100644 index 000000000..d9e833936 --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml @@ -0,0 +1,69 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + scm_sfc_flux_spec + dcyc2t3 + GFS_suite_interstitial_2 + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index 63d35c97c..f3420cdf5 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -44,6 +44,7 @@ def timestep(self, value): suite_list = [] suite_list.append(suite('SCM_GFS_v16', 'tracers_GFS_v16.txt', 'input_GFS_v16.nml', 600.0, 1800.0, True )) suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) +suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) From 84e30f4d047954ca55995b8688b3ffde180ae497 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 11 Apr 2024 16:17:10 -0400 Subject: [PATCH 148/189] merge changes from @hertneky in #446 --- scm/etc/Cheyenne_setup_gnu.csh | 55 ------------------------------ scm/etc/Cheyenne_setup_gnu.sh | 47 ------------------------- scm/etc/Cheyenne_setup_intel.csh | 55 ------------------------------ scm/etc/Cheyenne_setup_intel.sh | 47 ------------------------- scm/etc/Desktop_setup_gfortran.csh | 51 --------------------------- scm/etc/Desktop_setup_gfortran.sh | 47 ------------------------- scm/etc/Hera_setup_intel.csh | 47 ------------------------- scm/etc/Hera_setup_intel.sh | 39 --------------------- scm/etc/modules/hera_gnu.lua | 4 +-- scm/etc/modules/hera_intel.lua | 4 +-- 10 files changed, 4 insertions(+), 392 deletions(-) delete mode 100755 scm/etc/Cheyenne_setup_gnu.csh delete mode 100755 scm/etc/Cheyenne_setup_gnu.sh delete mode 100755 scm/etc/Cheyenne_setup_intel.csh delete mode 100755 scm/etc/Cheyenne_setup_intel.sh delete mode 100755 scm/etc/Desktop_setup_gfortran.csh delete mode 100755 scm/etc/Desktop_setup_gfortran.sh delete mode 100755 scm/etc/Hera_setup_intel.csh delete mode 100755 scm/etc/Hera_setup_intel.sh diff --git a/scm/etc/Cheyenne_setup_gnu.csh b/scm/etc/Cheyenne_setup_gnu.csh deleted file mode 100755 index c37acbdc2..000000000 --- a/scm/etc/Cheyenne_setup_gnu.csh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Cheyenne with gcc/gfortran" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading gnu and netcdf modules..." -module load ncarenv/1.3 -module load gnu/11.2.0 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack -module load hpc/1.2.0 -module load hpc-gnu/11.2.0 -module load hpc-mpt/2.25 -setenv bacio_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/bacio/2.4.1 -setenv sp_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/sp/2.3.3 -setenv w3emc_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC gcc -setenv CXX g++ -setenv FC gfortran - -echo "Loading cmake" -module load cmake/3.16.4 -setenv CMAKE_C_COMPILER gcc -setenv CMAKE_CXX_COMPILER g++ -setenv CMAKE_Fortran_COMPILER gfortran -setenv CMAKE_Platform cheyenne.gnu - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_gnu.sh b/scm/etc/Cheyenne_setup_gnu.sh deleted file mode 100755 index e1a190fcc..000000000 --- a/scm/etc/Cheyenne_setup_gnu.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Cheyenne with gcc/gfortran" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading gnu and netcdf modules..." -module load ncarenv/1.3 -module load gnu/11.2.0 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack -module load hpc/1.2.0 -module load hpc-gnu/11.2.0 -module load hpc-mpt/2.25 -export bacio_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/bacio/2.4.1 -export sp_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/sp/2.3.3 -export w3emc_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=gcc -export CXX=g++ -export FC=gfortran - -echo "Loading cmake" -module load cmake/3.22.0 -export CMAKE_C_COMPILER=gcc -export CMAKE_CXX_COMPILER=g++ -export CMAKE_Fortran_COMPILER=gfortran -export CMAKE_Platform=cheyenne.gnu - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_intel.csh b/scm/etc/Cheyenne_setup_intel.csh deleted file mode 100755 index d8bc17890..000000000 --- a/scm/etc/Cheyenne_setup_intel.csh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Cheyenne with icc/ifort" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module load ncarenv/1.3 -module load intel/2022.1 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1 -module load hpc-mpt/2.25 -setenv bacio_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/bacio/2.4.1 -setenv sp_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/sp/2.3.3 -setenv w3emc_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC icc -setenv CXX icpc -setenv FC ifort - -echo "Loading cmake" -module load cmake/3.22.0 -setenv CMAKE_C_COMPILER icc -setenv CMAKE_CXX_COMPILER icpc -setenv CMAKE_Fortran_COMPILER ifort -setenv CMAKE_Platform cheyenne.intel - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_intel.sh b/scm/etc/Cheyenne_setup_intel.sh deleted file mode 100755 index 71727e034..000000000 --- a/scm/etc/Cheyenne_setup_intel.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Cheyenne with icc/ifort" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module load ncarenv/1.3 -module load intel/2022.1 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1 -module load hpc-mpt/2.25 -export bacio_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/bacio/2.4.1 -export sp_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/sp/2.3.3 -export w3emc_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=icc -export CXX=icpc -export FC=ifort - -echo "Loading cmake" -module load cmake/3.22.0 -export CMAKE_C_COMPILER=icc -export CMAKE_CXX_COMPILER=icpc -export CMAKE_Fortran_COMPILER=ifort -export CMAKE_Platform=cheyenne.intel - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Desktop_setup_gfortran.csh b/scm/etc/Desktop_setup_gfortran.csh deleted file mode 100755 index d038405f4..000000000 --- a/scm/etc/Desktop_setup_gfortran.csh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Desktop (MacOS) with gcc/gfortran" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -echo "Setting CC/CXX/FC environment variables" -setenv CC /opt/local/bin/gcc-mp-10 -setenv CXX /opt/local/bin/g++-mp-10 -setenv FC gfortran-mp-10 - -echo "Setting location of NETCDF" -setenv NETCDF /opt/local -setenv LDFLAGS "-I${NETCDF}/include -L${NETCDF}/lib -Wl,-rpath,${NETCDF}/lib" - -echo "Setting location of NCEPLIBS libraries" -setenv BACIO_LIB4 /Users/$USER/NCEPLIBS/lib/libbacio_v2.2.0_4.a -setenv SP_LIBd /Users/$USER/NCEPLIBS/lib/libsp_v2.1.0_d.a -setenv W3NCO_LIBd /Users/$USER/NCEPLIBS/lib/libw3nco_v2.1.0_d.a - -#check to see if CMake is installed locally -echo "Checking if CMake is installed" -cmake --version - -if ( $? != 0 ) then - echo "CMake not found; installing CMake" - pip install cmake -else - echo "CMake is installed" -endif - -#check to see if f90nml is installed locally -echo "Checking if f90nml python module is installed" -python -c "import f90nml" - -if ( $? != 0 ) then - echo "Not found; installing f90nml" - pip install --no-cache-dir f90nml==0.19 -else - echo "f90nml is installed" -endif diff --git a/scm/etc/Desktop_setup_gfortran.sh b/scm/etc/Desktop_setup_gfortran.sh deleted file mode 100755 index 37485a163..000000000 --- a/scm/etc/Desktop_setup_gfortran.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Desktop (MacOS) with gcc/gfortran" - -if [[ $(uname -s) == Darwin ]]; then - MYDIR=$(cd "$(dirname "$(greadlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -else - MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -fi - -export SCM_ROOT=$MYDIR/../.. - -echo "Setting CC/CXX/FC environment variables" -export CC=/opt/local/bin/gcc-mp-10 -export CXX=/opt/local/bin/g++-mp-10 -export FC=gfortran-mp-10 - -echo "Setting location of NETCDF" -export NETCDF=/opt/local -export LDFLAGS="-I${NETCDF}/include -L${NETCDF}/lib -Wl,-rpath,${NETCDF}/lib" - -echo "Setting location of NCEPLIBS libraries" -export BACIO_LIB4=/Users/$USER/NCEPLIBS/lib/libbacio_v2.2.0_4.a -export SP_LIBd=/Users/$USER/NCEPLIBS/lib/libsp_v2.1.0_d.a -export W3NCO_LIBd=/Users/$USER/NCEPLIBS/lib/libw3nco_v2.1.0_d.a - -#check to see if CMake is installed locally -echo "Checking if CMake is installed" -cmake --version - -if [ $? -ne 0 ]; then - echo "CMake not found; installing CMake" - pip install cmake -else - echo "CMake is installed" -fi - -#check to see if f90nml is installed locally -echo "Checking if f90nml python module is installed" -python -c "import f90nml" - -if [ $? -ne 0 ]; then - echo "Not found; installing f90nml" - pip install --no-cache-dir f90nml==0.19 -else - echo "f90nml is installed" -fi diff --git a/scm/etc/Hera_setup_intel.csh b/scm/etc/Hera_setup_intel.csh deleted file mode 100755 index 369f6e3af..000000000 --- a/scm/etc/Hera_setup_intel.csh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Hera with icc/ifort" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module purge -module load intel/2022.1.2 -module load impi/2022.1.2 -module use /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1.2 -module load hpc-impi/2022.1.2 -module load netcdf - -echo "Setting up NCEPLIBS" -setenv bacio_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/bacio/2.4.1 -setenv sp_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/sp/2.3.3 -setenv w3emc_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC icc -setenv CXX icpc -setenv FC ifort - -echo "Loading cmake" -module load cmake/3.20.1 -setenv CMAKE_C_COMPILER icc -setenv CMAKE_CXX_COMPILER icpc -setenv CMAKE_Fortran_COMPILER ifort -setenv CMAKE_Platform hera.intel - -echo "Loading the SCM python environment" -source /scratch1/BMC/gmtb/SCM_anaconda/etc/profile.d/conda.csh -conda activate pyccpp diff --git a/scm/etc/Hera_setup_intel.sh b/scm/etc/Hera_setup_intel.sh deleted file mode 100755 index 5d760260b..000000000 --- a/scm/etc/Hera_setup_intel.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Hera with icc/ifort" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module purge -module load intel/2022.1.2 -module load impi/2022.1.2 -module use /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1.2 -module load hpc-impi/2022.1.2 -module load netcdf - -echo "Setting up NCEPLIBS" -export bacio_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/bacio/2.4.1 -export sp_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/sp/2.3.3 -export w3emc_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=icc -export CXX=icpc -export FC=ifort - -echo "Loading cmake" -module load cmake/3.20.1 -export CMAKE_C_COMPILER=icc -export CMAKE_CXX_COMPILER=icpc -export CMAKE_Fortran_COMPILER=ifort -export CMAKE_Platform=hera.intel - -echo "Loading the SCM python environment" -. "/scratch1/BMC/gmtb/SCM_anaconda/etc/profile.d/conda.sh" -conda activate pyccpp diff --git a/scm/etc/modules/hera_gnu.lua b/scm/etc/modules/hera_gnu.lua index 23867c4ac..0e28df2d0 100644 --- a/scm/etc/modules/hera_gnu.lua +++ b/scm/etc/modules/hera_gnu.lua @@ -7,10 +7,10 @@ whatis([===[Loads libraries needed for building the CCPP SCM on Hera with GNU co prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load("cmake/3.20.1") +load("cmake/3.28.1") load("miniconda/3.9.12") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core") load("stack-gcc/9.2.0") load("stack-openmpi/4.1.5") diff --git a/scm/etc/modules/hera_intel.lua b/scm/etc/modules/hera_intel.lua index d84db665c..6cdbf7d4c 100644 --- a/scm/etc/modules/hera_intel.lua +++ b/scm/etc/modules/hera_intel.lua @@ -6,10 +6,10 @@ the NOAA RDHPC machine Hera using Intel-2021.5.0 whatis([===[Loads libraries needed for building the CCPP SCM on Hera with Intel compilers ]===]) prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load("cmake/3.20.1") +load("cmake/3.28.1") load("miniconda/3.9.12") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core") load("stack-intel/2021.5.0") load("stack-intel-oneapi-mpi/2021.5.1") From 906e1e778e881d58b5004fc53d31d2ac7d9ba736 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 11 Apr 2024 16:26:22 -0400 Subject: [PATCH 149/189] add CITATION.cff file from @scrasmussen from #458 --- CITATION.cff | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..4c335023f --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,54 @@ +title: "CCPP Single Column Model (SCM)" +version: 6.0.0 +message: "Please cite this software using the metadata from this file." +type: software +identifiers: + - description: "This is the archived snapshot of all versions of CCPP SCM" + type: doi + value: 10.5281/zenodo.6896437 + - description: "This is the archived snapshot of CCPP SCM v6.0.0" + type: doi + value: 10.5281/zenodo.6896438 +repository-code: "https://github.com/NCAR/ccpp-scm" +url: "https://dtcenter.org/community-code/common-community-physics-package-ccpp" +authors: + - given-names: Grant + family-names: Firl + affiliation: >- + Cooperative Institute for Research in the Atmosphere, + Developmental Testbed Center, + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0009-0001-7076-2735 + - given-names: Dustin + family-names: Swales + affiliation: >- + Developmental Testbed Center and + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0000-0002-5322-4521 + - given-names: Laurie + family-names: Carson + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Ligia + family-names: Bernardet + affiliation: >- + Developmental Testbed Center and + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0000-0002-4952-4038 + - given-names: Dominikus + family-names: Heinzeller + affiliation: Joint Center for Satellite Data Assimilation + orcid: 0000-0003-2962-1049 + - given-names: Michelle + family-names: Harrold + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Tracy + family-names: Hertneky + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Michael + family-names: Kavulich + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research +cff-version: 1.2.0 From e5ff0720cf9426f7710b6ec82eae817ab18d99f7 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 11 Apr 2024 16:32:00 -0400 Subject: [PATCH 150/189] merge in changes from #428 from @hertneky --- scm/src/run_scm.py | 8 ++++---- scm/src/scm_input.F90 | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 20cfbce6a..926fb68ba 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -488,12 +488,12 @@ def setup_rundir(self): try: input_type = case_nml['case_config']['input_type'] if input_type == 1: + surface_flux_spec = False #open the case data file and read the surfaceForcing global attribute - case_data_dir = case_nml['case_config']['case_data_dir'] - nc_fid = Dataset(os.path.join(SCM_ROOT, case_data_dir) + '/' + self._case + '_SCM_driver.nc' , 'r') - surfaceForcing = nc_fid.getncattr('surfaceForcing') + nc_fid = Dataset(os.path.join(SCM_ROOT, self._case_data_dir) + '/' + self._case + '_SCM_driver.nc' , 'r') + surfaceForcing = nc_fid.getncattr('surface_forcing_temp') nc_fid.close() - if (surfaceForcing.lower() == 'flux' or surfaceForcing.lower() == 'surfaceflux'): + if (surfaceForcing.lower() == 'flux' or surfaceForcing.lower() == 'surface_flux'): surface_flux_spec = True except KeyError: # if not using DEPHY format, check to see if surface fluxes are specified in the case configuration file (default is False) diff --git a/scm/src/scm_input.F90 b/scm/src/scm_input.F90 index 3269c16ac..cc3effcd5 100644 --- a/scm/src/scm_input.F90 +++ b/scm/src/scm_input.F90 @@ -950,7 +950,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) real(kind=sp) :: z_nudging_temp, z_nudging_theta, z_nudging_thetal, z_nudging_qv, z_nudging_qt, z_nudging_rv, z_nudging_rt, z_nudging_u, z_nudging_v real(kind=sp) :: p_nudging_temp, p_nudging_theta, p_nudging_thetal, p_nudging_qv, p_nudging_qt, p_nudging_rv, p_nudging_rt, p_nudging_u, p_nudging_v character(len=5) :: input_surfaceType - character(len=11) :: input_surfaceForcingWind='',input_surfaceForcingMoist='',input_surfaceForcingLSM='',input_surfaceForcingTemp='' + character(len=12) :: input_surfaceForcingWind='',input_surfaceForcingMoist='',input_surfaceForcingLSM='',input_surfaceForcingTemp='' ! initial variables (IC = Initial Condition) real(kind=dp), allocatable :: input_lat(:) !< column latitude (deg) @@ -1630,7 +1630,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) call NetCDF_read_var(ncid, "wprvp_s", .False., input_force_wprvp) call NetCDF_read_var(ncid, "wprtp_s", .False., input_force_wprtp) else if (trim(input_surfaceForcingMoist) == 'surface_flux') then - call NetCDF_read_var(ncid, "hfls", .False., input_force_sfc_sens_flx) + call NetCDF_read_var(ncid, "hfls", .False., input_force_sfc_lat_flx) endif ! @@ -1969,7 +1969,6 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) if (trim(input_surfaceForcingLSM) == "lsm") then scm_input%input_ozone = input_ozone(:,active_init_time) scm_input%input_area = input_area(active_init_time) - scm_state%area = input_area(active_init_time) scm_input%input_stddev = input_stddev(active_init_time) scm_input%input_convexity= input_convexity(active_init_time) From 680368def0d3ae64450717bd28ff51ba759d3539 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 1 May 2024 17:05:42 -0400 Subject: [PATCH 151/189] add script to download and stage climatological aerosol data and fix typo in HR3 namelist --- ...t_GFS_v17_H3.nml => input_GFS_v17_HR3.nml} | 2 +- contrib/get_aerosol_climo.sh | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) rename ccpp/physics_namelists/{input_GFS_v17_H3.nml => input_GFS_v17_HR3.nml} (99%) create mode 100755 contrib/get_aerosol_climo.sh diff --git a/ccpp/physics_namelists/input_GFS_v17_H3.nml b/ccpp/physics_namelists/input_GFS_v17_HR3.nml similarity index 99% rename from ccpp/physics_namelists/input_GFS_v17_H3.nml rename to ccpp/physics_namelists/input_GFS_v17_HR3.nml index 10d60a77e..a4761fc3e 100644 --- a/ccpp/physics_namelists/input_GFS_v17_H3.nml +++ b/ccpp/physics_namelists/input_GFS_v17_HR3.nml @@ -43,7 +43,7 @@ fhlwr = 1200. ialb = 1 iems = 1 - iaer = 5111 + iaer = 1011 icliq_sw = 2 ico2 = 2 isubc_sw = 2 diff --git a/contrib/get_aerosol_climo.sh b/contrib/get_aerosol_climo.sh new file mode 100755 index 000000000..2039dc732 --- /dev/null +++ b/contrib/get_aerosol_climo.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +#set -ex + +# Directory where this script is located +if [[ $(uname -s) == Darwin ]]; then + if [[ $(sw_vers -productVersion) < 12.3 ]]; then + MYDIR=$(cd "$(dirname "$(greadlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) + else + MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) + fi +else + MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) +fi +BASEDIR=$MYDIR/.. + +# Change to directory containing the physics input data, download and extract archive +data_files=("FV3_aeroclim1" "FV3_aeroclim2" "FV3_aeroclim3" "FV3_aeroclim_optics") + +cd $BASEDIR/scm/data/physics_input_data/ +for file in "${data_files[@]}"; do + echo "Retrieving $file.tar.gz" + wget https://github.com/NCAR/ccpp-scm/releases/download/v6.0.0/${file}.tar.gz + tar -xvf ${file}.tar.gz + rm -f ${file}.tar.gz +done + +cd $BASEDIR/ + From 7ad487a426f92057905333b05accb040eefaeb0b Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 2 May 2024 16:45:41 -0400 Subject: [PATCH 152/189] add SCM_RRFS_v1 SDF, nml, tracer file; update suite_info; update scm_type_defs to recgonize new tracers --- ccpp/physics_namelists/input_RRFS_v1.nml | 145 ++++++++++++++++++++++ ccpp/suites/suite_SCM_RRFS_v1.xml | 79 ++++++++++++ ccpp/suites/suite_SCM_RRFS_v1_ps.xml | 63 ++++++++++ scm/etc/tracer_config/tracers_RRFS_v1.txt | 16 +++ scm/src/scm_type_defs.F90 | 6 + scm/src/suite_info.py | 1 + 6 files changed, 310 insertions(+) create mode 100644 ccpp/physics_namelists/input_RRFS_v1.nml create mode 100644 ccpp/suites/suite_SCM_RRFS_v1.xml create mode 100644 ccpp/suites/suite_SCM_RRFS_v1_ps.xml create mode 100644 scm/etc/tracer_config/tracers_RRFS_v1.txt diff --git a/ccpp/physics_namelists/input_RRFS_v1.nml b/ccpp/physics_namelists/input_RRFS_v1.nml new file mode 100644 index 000000000..60de83a4e --- /dev/null +++ b/ccpp/physics_namelists/input_RRFS_v1.nml @@ -0,0 +1,145 @@ +&gfs_physics_nml + addsmoke_flag = 1 + aero_dir_fdb = .true. + aero_ind_fdb = .false. + bl_mynn_edmf = 1 + bl_mynn_edmf_mom = 1 + bl_mynn_tkeadvect = .true. + cal_pre = .false. + cdmbgwd = 3.5, 1.0 + clm_debug_print = .false. + clm_lake_debug = .false. + cnvcld = .false. + cnvgwd = .false. + coarsepm_settling = 1 + cplflx = .false. + diag_log = .true. + debug = .false. + do_deep = .true. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_mynnedmf = .true. + do_mynnsfclay = .true. + do_plumerise = .true. + do_smoke_transport = .true. + do_tofd = .false. + do_ugwp = .false. + do_ugwp_v0 = .false. + do_ugwp_v0_nst_only = .false. + do_ugwp_v0_orog_only = .false. + drydep_opt = 1 + dspheat = .true. + dt_inner = 36 + dust_alpha = 10.0 + dust_drylimit_factor = 0.5 + dust_gamma = 1.3 + dust_moist_correction = 2.0 + dust_opt = 1 + ebb_dcycle = 2 + effr_in = .true. + enh_mix = .false. + fhcyc = 0 + fhlwr = 900.0 + fhswr = 900.0 + fhzero = 1.0 + frac_ice = .true. + gwd_opt = 3 + h2o_phys = .true. + hybedmf = .false. + iaer = 1011 + ialb = 2 + iau_delthrs = 6 + iau_inc_files = '' + iaufhrs = 30 + iccn = 2 + icliq_sw = 2 + icloud_bl = 1 + ico2 = 2 + iems = 2 + imfdeepcnv = 3 + imfshalcnv = -1 + imp_physics = 8 + iopt_alb = 2 + iopt_btr = 1 + iopt_crs = 1 + iopt_dveg = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_lake = 2 + iopt_rad = 1 + iopt_run = 1 + iopt_sfc = 1 + iopt_snf = 4 + iopt_stc = 1 + iopt_tbot = 2 + iovr = 3 + isncond_opt = 2 + isncovr_opt = 3 + isol = 2 + isot = 1 + isubc_lw = 2 + isubc_sw = 2 + ivegsrc = 1 + kice = 9 + ldiag3d = .true. + ldiag_ugwp = .false. + lgfdlmprad = .false. + lheatstrg = .false. + lightning_threat = .true. + lkm = 1 + lradar = .true. + lrefres = .true. + lsm = 3 + lsoil = 9 + lsoil_lsm = 9 + ltaerosol = .true. + lwhtr = .true. + min_lakeice = 0.15 + min_seaice = 0.15 + mix_chem = .true. + mosaic_lu = 1 + mosaic_soil = 1 + nsfullradar_diag = 3600 + oz_phys = .false. + oz_phys_2015 = .true. + pdfcld = .false. + plume_wind_eff = 1 + plumerisefire_frq = 60 + pre_rad = .false. + print_diff_pgr = .true. + prslrd0 = 0.0 + qdiag3d = .true. + random_clds = .false. + redrag = .true. + rrfs_sd = .true. + rrfs_smoke_debug = .false. + satmedmf = .false. + seas_opt = 0 + sfclay_compute_flux = .true. + shal_cnv = .false. + smoke_conv_wet_coef = 0.5, 0.5, 0.5 + smoke_forecast = 1 + swhtr = .true. + thsfc_loc = .false. + trans_trac = .true. + ttendlim = -999 + use_ufo = .true. + wetdep_ls_alpha = 0.5 + wetdep_ls_opt = 1 +/ + +&cires_ugwp_nml + knob_ugwp_azdir = 2, 4, 4, 4 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 1 + knob_ugwp_effac = 1, 1, 1, 1 + knob_ugwp_ndx4lh = 1 + knob_ugwp_solver = 2 + knob_ugwp_source = 1, 1, 0, 0 + knob_ugwp_stoch = 0, 0, 0, 0 + knob_ugwp_version = 0 + knob_ugwp_wvspec = 1, 25, 25, 25 + launch_level = 25 +/ diff --git a/ccpp/suites/suite_SCM_RRFS_v1.xml b/ccpp/suites/suite_SCM_RRFS_v1.xml new file mode 100644 index 000000000..ef1c73157 --- /dev/null +++ b/ccpp/suites/suite_SCM_RRFS_v1.xml @@ -0,0 +1,79 @@ + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + sgscloud_radpre + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + sgscloud_radpost + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + mynnsfc_wrapper + GFS_surface_loop_control_part1 + lsm_ruc + clm_lake + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + rrfs_smoke_wrapper + mynnedmf_wrapper + rrfs_smoke_postpbl + GFS_GWD_generic_pre + drag_suite + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + cu_gf_driver_pre + cu_gf_driver + GFS_DCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + mp_thompson + mp_thompson_post + GFS_MP_generic_post + cu_gf_driver_post + maximum_hourly_diagnostics + GFS_physics_post + + + \ No newline at end of file diff --git a/ccpp/suites/suite_SCM_RRFS_v1_ps.xml b/ccpp/suites/suite_SCM_RRFS_v1_ps.xml new file mode 100644 index 000000000..91918a6b8 --- /dev/null +++ b/ccpp/suites/suite_SCM_RRFS_v1_ps.xml @@ -0,0 +1,63 @@ + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + sgscloud_radpre + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + sgscloud_radpost + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + scm_sfc_flux_spec + dcyc2t3 + GFS_suite_interstitial_2 + rrfs_smoke_wrapper + mynnedmf_wrapper + rrfs_smoke_postpbl + GFS_GWD_generic_pre + drag_suite + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + cu_gf_driver_pre + cu_gf_driver + GFS_DCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + mp_thompson + mp_thompson_post + GFS_MP_generic_post + cu_gf_driver_post + maximum_hourly_diagnostics + GFS_physics_post + + + \ No newline at end of file diff --git a/scm/etc/tracer_config/tracers_RRFS_v1.txt b/scm/etc/tracer_config/tracers_RRFS_v1.txt new file mode 100644 index 000000000..3835c38a2 --- /dev/null +++ b/scm/etc/tracer_config/tracers_RRFS_v1.txt @@ -0,0 +1,16 @@ +"sphum","water_vapor_specific_humidity","kg kg-1" +"liq_wat","cloud_condensed_water_mixing_ratio","kg kg-1" +"ice_wat","ice_water_mixing_ratio","kg kg-1" +"rainwat","rain_water_mixing_ratio","kg kg-1" +"snowwat","snow_water_mixing_ratio","kg kg-1" +"graupel","graupel_mixing_ratio","kg kg-1" +"water_nc","cloud_droplet_number_concentration","kg-1" +"ice_nc","ice_number_concentration","kg-1" +"rain_nc","rain_number_concentration","kg-1" +"o3mr","ozone_mixing_ratio","kg kg-1" +"liq_aero","water_friendly_aerosol_number_concentration","kg-1" +"ice_aero","ice_friendly_aerosol_number_concentration","kg-1" +"sgs_tke","turbulent_kinetic_energy","m2 s-2" +"smoke","smoke_tracer_concentration","ug kg-1" +"dust","dust_tracer_concentration","ug kg-1" +"coarsepm","coarse_particulate_matter","ug kg-1" \ No newline at end of file diff --git a/scm/src/scm_type_defs.F90 b/scm/src/scm_type_defs.F90 index 550f7d951..ce9805ae6 100644 --- a/scm/src/scm_type_defs.F90 +++ b/scm/src/scm_type_defs.F90 @@ -85,6 +85,9 @@ module scm_type_defs integer :: water_friendly_aerosol_index !< index for water-friendly aerosols in the tracer array integer :: ice_friendly_aerosol_index !< index for ice-friendly aerosols in the tracer array integer :: mass_weighted_rime_factor_index !< index for mass-weighted rime factor + integer :: smoke_index !< index for smoke in the tracer array + integer :: dust_index !< index for dust in the tracer array + integer :: coarsepm_index !< index for coarsepm in the tracer array integer :: init_year, init_month, init_day, init_hour, init_min character(len=32), allocatable :: tracer_names(:) !< integer, allocatable :: tracer_types(:) !< @@ -489,6 +492,9 @@ subroutine scm_state_create(scm_state, n_columns, n_levels, n_soil, n_snow, n_ti scm_state%water_friendly_aerosol_index = get_tracer_index(scm_state%tracer_names,"liq_aero") scm_state%ice_friendly_aerosol_index = get_tracer_index(scm_state%tracer_names,"ice_aero") scm_state%mass_weighted_rime_factor_index = get_tracer_index(scm_state%tracer_names,"q_rimef") + scm_state%smoke_index = get_tracer_index(scm_state%tracer_names,"smoke") + scm_state%dust_index = get_tracer_index(scm_state%tracer_names,"dust") + scm_state%coarsepm_index = get_tracer_index(scm_state%tracer_names,"coarsepm") scm_state%nwat = 0 if(scm_state%water_vapor_index /= -99) then diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index f3420cdf5..19dffb022 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -46,6 +46,7 @@ def timestep(self, value): suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) +suite_list.append(suite('SCM_RRFS_v1', 'tracers_RRFS_v1.txt', 'input_RRFS_v1.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_HRRR', 'tracers_HRRR.txt', 'input_HRRR.nml', 600.0, 600.0 , True )) From 0d294b3bb03c5f60941ea34d610fd6b718ea937e Mon Sep 17 00:00:00 2001 From: "Man.Zhang" Date: Wed, 15 May 2024 10:40:51 -0600 Subject: [PATCH 153/189] Add namelist option and sigmab (tracer) for HR3 --- ...t_GFS_v17_H3.nml => input_GFS_v17_HR3.nml} | 32 ++-- .../input_GFS_v17_HR3_ps.nml | 171 ++++++++++++++++++ scm/etc/tracer_config/tracers_GFS_v17_HR3.txt | 11 ++ scm/src/GFS_typedefs.F90 | 1 + scm/src/scm_output.F90 | 3 + scm/src/scm_type_defs.F90 | 2 + scm/src/suite_info.py | 2 +- scm/src/supported_suites.py | 2 +- 8 files changed, 206 insertions(+), 18 deletions(-) rename ccpp/physics_namelists/{input_GFS_v17_H3.nml => input_GFS_v17_HR3.nml} (90%) create mode 100644 ccpp/physics_namelists/input_GFS_v17_HR3_ps.nml create mode 100644 scm/etc/tracer_config/tracers_GFS_v17_HR3.txt diff --git a/ccpp/physics_namelists/input_GFS_v17_H3.nml b/ccpp/physics_namelists/input_GFS_v17_HR3.nml similarity index 90% rename from ccpp/physics_namelists/input_GFS_v17_H3.nml rename to ccpp/physics_namelists/input_GFS_v17_HR3.nml index 10d60a77e..8d859e580 100644 --- a/ccpp/physics_namelists/input_GFS_v17_H3.nml +++ b/ccpp/physics_namelists/input_GFS_v17_HR3.nml @@ -12,7 +12,7 @@ ltaerosol = .false. lradar = .true. ttendlim = -999 - dt_inner = 225.0 + dt_inner = 150 sedi_semi = .true. decfl = 10 oz_phys = .false. @@ -36,14 +36,18 @@ do_gsl_drag_tofd = .true. do_ugwp_v1_orog_only = .false. min_lakeice = 0.15 - min_seaice = 0.15 - use_cice_alb = .false. + min_seaice = 1.0e-6 + use_cice_alb = .true. pdfcld = .false. fhswr = 1200. fhlwr = 1200. - ialb = 1 - iems = 1 - iaer = 5111 + progsigma = .true. + betascu = 8.0 + betamcu = 1.0 + betadcu = 2.0 + ialb = 2 + iems = 2 + iaer = 1011 icliq_sw = 2 ico2 = 2 isubc_sw = 2 @@ -67,7 +71,7 @@ imfshalcnv = 2 imfdeepcnv = 2 ras = .false. - cdmbgwd = 10.0,3.5,1.0,1.0 + cdmbgwd = 2.5,7.5,1.0,1.0 prslrd0 = 0. ivegsrc = 1 isot = 1 @@ -87,14 +91,13 @@ iopt_tbot = 2 iopt_stc = 3 debug = .false. - nstf_name = 2,1,0,0,0 + nstf_name = 2,0,0,0,0 nst_anl = .true. psautco = 0.0008,0.0005 prautco = 0.00015,0.00015 lgfdlmprad = .false. effr_in = .true. ldiag_ugwp = .false. - fscav_aero = "'*:0.0'" do_sppt = .false. do_shum = .false. do_skeb = .false. @@ -120,7 +123,7 @@ cplwav2atm = .false. do_ca = .false. ca_global = .false. - ca_sgs = .false. + ca_sgs = .true. nca = 1 ncells = 5 nlives = 12 @@ -129,10 +132,7 @@ nthresh = 18 ca_trigger = .true. nspinup = 1 - iseed_ca = 12345 - lndp_type = 0 - n_var_lndp = 0 - do_ccpp_suite_sim = .false. + iseed_ca = 1448371824 / &cires_ugwp_nml @@ -148,12 +148,12 @@ knob_ugwp_dokdis = 2 knob_ugwp_ndx4lh = 4 knob_ugwp_palaunch = 275.0e2 - knob_ugwp_nslope = 0 + knob_ugwp_nslope = 1 knob_ugwp_lzmax = 15.750e3 knob_ugwp_lzmin = 0.75e3 knob_ugwp_lzstar = 2.0e3 knob_ugwp_taumin = 0.25e-3 - knob_ugwp_tauamp = 1.5e-3 + knob_ugwp_tauamp = 0.5e-3 knob_ugwp_lhmet = 200.0e3 knob_ugwp_orosolv = 'pss-1986' / diff --git a/ccpp/physics_namelists/input_GFS_v17_HR3_ps.nml b/ccpp/physics_namelists/input_GFS_v17_HR3_ps.nml new file mode 100644 index 000000000..b2f93c79c --- /dev/null +++ b/ccpp/physics_namelists/input_GFS_v17_HR3_ps.nml @@ -0,0 +1,171 @@ +&gfs_physics_nml + fhzero = 6 + h2o_phys = .true. + ldiag3d = .true. + qdiag3d = .true. + print_diff_pgr = .false. + fhcyc = 24 + use_ufo = .true. + pre_rad = .false. + imp_physics = 8 + iovr = 3 + ltaerosol = .False. + lradar = .true. + ttendlim = -999 + dt_inner = 150 + sedi_semi = .true. + decfl = 10 + oz_phys = .false. + oz_phys_2015 = .true. + lsoil_lsm = 4 + do_mynnedmf = .false. + do_mynnsfclay = .false. + icloud_bl = 1 + bl_mynn_edmf = 1 + bl_mynn_tkeadvect = .true. + bl_mynn_edmf_mom = 1 + do_ugwp = .false. + do_tofd = .false. + gwd_opt = 2 + do_ugwp_v0 = .false. + do_ugwp_v1 = .true. + do_ugwp_v0_orog_only = .false. + do_ugwp_v0_nst_only = .false. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_ugwp_v1_orog_only = .false. + min_lakeice = 0.15 + min_seaice = 1.0e-6 + use_cice_alb = .true. + pdfcld = .false. + fhswr = 1200. + fhlwr = 1200. + progsigma = .true. + betascu = 8.0 + betamcu = 1.0 + betadcu = 2.0 + ialb = 2 + iems = 2 + iaer = 1011 + icliq_sw = 2 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + shal_cnv = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + satmedmf = .true. + isatmedmf = 1 + lheatstrg = .false. + lseaspray = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + imfshalcnv = 2 + imfdeepcnv = 2 + ras = .false. + cdmbgwd = 2.5,7.5,1.0,1.0 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = 2 + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = 3 + iopt_trs = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + debug = .false. + nstf_name = 2,0,0,0,0 + nst_anl = .true. + psautco = 0.0008,0.0005 + prautco = 0.00015,0.00015 + lgfdlmprad = .false. + effr_in = .true. + ldiag_ugwp = .false. + do_sppt = .false. + do_shum = .false. + do_skeb = .false. + do_RRTMGP = .false. + doGP_cldoptics_LUT = .true. + doGP_lwscat = .true. + active_gases = 'h2o_co2_o3_n2o_ch4_o2' + ngases = 6 + rrtmgp_root = '../../ccpp/physics/physics/rte-rrtmgp/' + lw_file_gas = 'rrtmgp/data/rrtmgp-data-lw-g128-210809.nc' + lw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-lw.nc' + sw_file_gas = 'rrtmgp/data/rrtmgp-data-sw-g112-210809.nc' + sw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-sw.nc' + rrtmgp_nGptsSW = 112 + rrtmgp_nGptsLW = 128 + rrtmgp_nBandsLW = 16 + rrtmgp_nBandsSW = 14 + frac_grid = .true. + cplchm = .false. + cplflx = .false. + cplice = .false. + cplwav = .false. + cplwav2atm = .false. + do_ca = .false. + ca_global = .false. + ca_sgs = .true. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + nthresh = 18 + ca_trigger = .true. + nspinup = 1 + iseed_ca = 1448371824 +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_version = 1 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 2 + knob_ugwp_ndx4lh = 4 + knob_ugwp_palaunch = 275.0e2 + knob_ugwp_nslope = 1 + knob_ugwp_lzmax = 15.750e3 + knob_ugwp_lzmin = 0.75e3 + knob_ugwp_lzstar = 2.0e3 + knob_ugwp_taumin = 0.25e-3 + knob_ugwp_tauamp = 0.5e-3 + knob_ugwp_lhmet = 200.0e3 + knob_ugwp_orosolv = 'pss-1986' +/ + +&ccpp_suite_sim_nml + suite_sim_file = '' + nprc_sim = 7 + prc_LWRAD_cfg = 0, 0, 1 + prc_SWRAD_cfg = 0, 0, 2 + prc_PBL_cfg = 1, 0, 3 + prc_GWD_cfg = 1, 0, 4 + prc_SCNV_cfg = 1, 1, 5 + prc_DCNV_cfg = 1, 1, 6 + prc_cldMP_cfg = 1, 1, 7 +/ diff --git a/scm/etc/tracer_config/tracers_GFS_v17_HR3.txt b/scm/etc/tracer_config/tracers_GFS_v17_HR3.txt new file mode 100644 index 000000000..eeba3d9e3 --- /dev/null +++ b/scm/etc/tracer_config/tracers_GFS_v17_HR3.txt @@ -0,0 +1,11 @@ +"sphum","water_vapor_specific_humidity","kg kg-1" +"liq_wat","cloud_condensed_water_mixing_ratio","kg kg-1" +"ice_wat","ice_water_mixing_ratio","kg kg-1" +"rainwat","rain_water_mixing_ratio","kg kg-1" +"snowwat","snow_water_mixing_ratio","kg kg-1" +"graupel","graupel_mixing_ratio","kg kg-1" +"ice_nc","ice_number_concentration","kg-1" +"rain_nc","rain_number_concentration","kg-1" +"o3mr","ozone_mixing_ratio","kg kg-1" +"sgs_tke","turbulent_kinetic_energy","m2 s-2" +"sigmab"," prognostic_updraft_area_fraction_in_convection","frac" diff --git a/scm/src/GFS_typedefs.F90 b/scm/src/GFS_typedefs.F90 index 2e822487a..cf4a6db0c 100644 --- a/scm/src/GFS_typedefs.F90 +++ b/scm/src/GFS_typedefs.F90 @@ -5393,6 +5393,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & call label_dtend_tracer(Model,100+Model%ntgz,'graupel_ref','graupel reflectivity','m3 kg-1 s-1') call label_dtend_tracer(Model,100+Model%nthz,'hail_ref','hail reflectivity','m3 kg-1 s-1') call label_dtend_tracer(Model,100+Model%ntke,'sgs_tke','turbulent kinetic energy','J s-1') + call label_dtend_tracer(Model,100+Model%ntsigma,'sigmab','prognostic updraft area fraction in convection','frac') call label_dtend_tracer(Model,100+Model%nqrimef,'q_rimef','mass weighted rime factor','kg-1 s-1') call label_dtend_tracer(Model,100+Model%ntwa,'liq_aero','number concentration of water-friendly aerosols','kg-1 s-1') call label_dtend_tracer(Model,100+Model%ntia,'ice_aero','number concentration of ice-friendly aerosols','kg-1 s-1') diff --git a/scm/src/scm_output.F90 b/scm/src/scm_output.F90 index 3e452882b..eac7470d2 100644 --- a/scm/src/scm_output.F90 +++ b/scm/src/scm_output.F90 @@ -187,6 +187,7 @@ subroutine output_init_state(ncid, time_inst_id, hor_dim_id, vert_dim_id, vert_d call NetCDF_def_var(ncid, 'ql', NF90_FLOAT, "suspended resolved liquid cloud water on model layer centers", "kg kg-1", dummy_id, (/ hor_dim_id, vert_dim_id, time_inst_id /)) call NetCDF_def_var(ncid, 'qi', NF90_FLOAT, "suspended resolved ice cloud water on model layer centers", "kg kg-1", dummy_id, (/ hor_dim_id, vert_dim_id, time_inst_id /)) call NetCDF_def_var(ncid, 'qc', NF90_FLOAT, "suspended (resolved + SGS) total cloud water on model layer centers", "kg kg-1", dummy_id, (/ hor_dim_id, vert_dim_id, time_inst_id /)) + call NetCDF_def_var(ncid, 'sigmab', NF90_FLOAT, "updraft area fraction on model layer centers", "frac", dummy_id, (/ hor_dim_id, time_inst_id /)) end subroutine output_init_state @@ -497,6 +498,8 @@ subroutine output_append_state(ncid, scm_state, physics) call NetCDF_put_var(ncid, "v", scm_state%state_v(:,:,1), scm_state%itt_out) call NetCDF_put_var(ncid, "ql", scm_state%state_tracer(:,:,scm_state%cloud_water_index,1), scm_state%itt_out) call NetCDF_put_var(ncid, "qi", scm_state%state_tracer(:,:,scm_state%cloud_ice_index,1), scm_state%itt_out) + !zhang + call NetCDF_put_var(ncid, "sigmab", scm_state%state_tracer(:,1,scm_state%sigmab_index,1), scm_state%itt_out) if (physics%model%do_mynnedmf) then call NetCDF_put_var(ncid, "qc", scm_state%state_tracer(:,:,scm_state%cloud_water_index,1) + & scm_state%state_tracer(:,:,scm_state%cloud_ice_index,1) + & diff --git a/scm/src/scm_type_defs.F90 b/scm/src/scm_type_defs.F90 index 550f7d951..a84d22a7b 100644 --- a/scm/src/scm_type_defs.F90 +++ b/scm/src/scm_type_defs.F90 @@ -81,6 +81,7 @@ module scm_type_defs integer :: graupel_volume_index !< index for graupel volume in the tracer array integer :: hail_volume_index !< index for hail volume in the tracer array integer :: tke_index !< index for TKE in the tracer array + integer :: sigmab_index !< index for prognostic updraft area fraction in convection integer :: ccn_index !< index for CCN in the tracer array integer :: water_friendly_aerosol_index !< index for water-friendly aerosols in the tracer array integer :: ice_friendly_aerosol_index !< index for ice-friendly aerosols in the tracer array @@ -485,6 +486,7 @@ subroutine scm_state_create(scm_state, n_columns, n_levels, n_soil, n_snow, n_ti scm_state%graupel_volume_index = get_tracer_index(scm_state%tracer_names,"graupel_vol") scm_state%hail_volume_index = get_tracer_index(scm_state%tracer_names,"hail_vol") scm_state%tke_index = get_tracer_index(scm_state%tracer_names,"sgs_tke") + scm_state%sigmab_index = get_tracer_index(scm_state%tracer_names,"sigmab") scm_state%ccn_index = get_tracer_index(scm_state%tracer_names,"ccn_nc") scm_state%water_friendly_aerosol_index = get_tracer_index(scm_state%tracer_names,"liq_aero") scm_state%ice_friendly_aerosol_index = get_tracer_index(scm_state%tracer_names,"ice_aero") diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index f3420cdf5..87470f6d4 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -44,7 +44,7 @@ def timestep(self, value): suite_list = [] suite_list.append(suite('SCM_GFS_v16', 'tracers_GFS_v16.txt', 'input_GFS_v16.nml', 600.0, 1800.0, True )) suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) -suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) +suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_HR3_aero.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) diff --git a/scm/src/supported_suites.py b/scm/src/supported_suites.py index 8cbac028c..aee06a705 100644 --- a/scm/src/supported_suites.py +++ b/scm/src/supported_suites.py @@ -1 +1 @@ -suites = ["SCM_GFS_v16","SCM_GFS_v17_p8","SCM_HRRR","SCM_RAP","SCM_RRFS_v1beta","SCM_WoFS_v0"] +suites = ["SCM_GFS_v16","SCM_GFS_v17_p8","SCM_GFS_v17_HR3","SCM_HRRR","SCM_RAP","SCM_RRFS_v1beta","SCM_WoFS_v0"] From e3869e11abd027e61884b637093377b495fa5d16 Mon Sep 17 00:00:00 2001 From: "Man.Zhang" Date: Wed, 15 May 2024 11:12:23 -0600 Subject: [PATCH 154/189] minor fix/cleanup --- scm/src/scm_output.F90 | 1 - scm/src/suite_info.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scm/src/scm_output.F90 b/scm/src/scm_output.F90 index eac7470d2..3458da3c9 100644 --- a/scm/src/scm_output.F90 +++ b/scm/src/scm_output.F90 @@ -498,7 +498,6 @@ subroutine output_append_state(ncid, scm_state, physics) call NetCDF_put_var(ncid, "v", scm_state%state_v(:,:,1), scm_state%itt_out) call NetCDF_put_var(ncid, "ql", scm_state%state_tracer(:,:,scm_state%cloud_water_index,1), scm_state%itt_out) call NetCDF_put_var(ncid, "qi", scm_state%state_tracer(:,:,scm_state%cloud_ice_index,1), scm_state%itt_out) - !zhang call NetCDF_put_var(ncid, "sigmab", scm_state%state_tracer(:,1,scm_state%sigmab_index,1), scm_state%itt_out) if (physics%model%do_mynnedmf) then call NetCDF_put_var(ncid, "qc", scm_state%state_tracer(:,:,scm_state%cloud_water_index,1) + & diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index 87470f6d4..fb60e12ae 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -44,7 +44,7 @@ def timestep(self, value): suite_list = [] suite_list.append(suite('SCM_GFS_v16', 'tracers_GFS_v16.txt', 'input_GFS_v16.nml', 600.0, 1800.0, True )) suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) -suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_HR3_aero.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) +suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_HR3.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) From cb1e5e264925dfba5a4faeff5225850d5f6b52f5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 20 May 2024 15:30:39 +0000 Subject: [PATCH 155/189] Add GP SDFs/NMLs to SCM --- .../input_GFS_v17_HR3_RRTMGP.nml | 171 ++++++++++++++++++ .../input_GFS_v17_HR3_RRTMGP_ps.nml | 171 ++++++++++++++++++ ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP.xml | 89 +++++++++ .../suite_SCM_GFS_v17_HR3_RRTMGP_ps.xml | 70 +++++++ scm/src/suite_info.py | 1 + 5 files changed, 502 insertions(+) create mode 100644 ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP.nml create mode 100644 ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP_ps.nml create mode 100644 ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP.xml create mode 100644 ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP_ps.xml diff --git a/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP.nml b/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP.nml new file mode 100644 index 000000000..f7cf90973 --- /dev/null +++ b/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP.nml @@ -0,0 +1,171 @@ +&gfs_physics_nml + fhzero = 6 + h2o_phys = .true. + ldiag3d = .true. + qdiag3d = .true. + print_diff_pgr = .false. + fhcyc = 24 + use_ufo = .true. + pre_rad = .false. + imp_physics = 8 + iovr = 3 + ltaerosol = .false. + lradar = .true. + ttendlim = -999 + dt_inner = 150 + sedi_semi = .true. + decfl = 10 + oz_phys = .false. + oz_phys_2015 = .true. + lsoil_lsm = 4 + do_mynnedmf = .false. + do_mynnsfclay = .false. + icloud_bl = 1 + bl_mynn_edmf = 1 + bl_mynn_tkeadvect = .true. + bl_mynn_edmf_mom = 1 + do_ugwp = .false. + do_tofd = .false. + gwd_opt = 2 + do_ugwp_v0 = .false. + do_ugwp_v1 = .true. + do_ugwp_v0_orog_only = .false. + do_ugwp_v0_nst_only = .false. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_ugwp_v1_orog_only = .false. + min_lakeice = 0.15 + min_seaice = 1.0e-6 + use_cice_alb = .true. + pdfcld = .false. + fhswr = 1200. + fhlwr = 1200. + progsigma = .true. + betascu = 8.0 + betamcu = 1.0 + betadcu = 2.0 + ialb = 2 + iems = 2 + iaer = 1011 + icliq_sw = 2 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + shal_cnv = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + satmedmf = .true. + isatmedmf = 1 + lheatstrg = .true. + lseaspray = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + imfshalcnv = 2 + imfdeepcnv = 2 + ras = .false. + cdmbgwd = 2.5,7.5,1.0,1.0 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = 2 + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = 3 + iopt_trs = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + debug = .false. + nstf_name = 2,0,0,0,0 + nst_anl = .true. + psautco = 0.0008,0.0005 + prautco = 0.00015,0.00015 + lgfdlmprad = .false. + effr_in = .true. + ldiag_ugwp = .false. + do_sppt = .false. + do_shum = .false. + do_skeb = .false. + do_RRTMGP = .true. + doGP_cldoptics_LUT = .true. + doGP_lwscat = .true. + active_gases = 'h2o_co2_o3_n2o_ch4_o2' + ngases = 6 + rrtmgp_root = '../../ccpp/physics/physics/Radiation/RRTMGP/rte-rrtmgp/' + lw_file_gas = 'rrtmgp/data/rrtmgp-data-lw-g128-210809.nc' + lw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-lw.nc' + sw_file_gas = 'rrtmgp/data/rrtmgp-data-sw-g112-210809.nc' + sw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-reordered-sw.nc' + rrtmgp_nGptsSW = 112 + rrtmgp_nGptsLW = 128 + rrtmgp_nBandsLW = 16 + rrtmgp_nBandsSW = 14 + frac_grid = .true. + cplchm = .false. + cplflx = .false. + cplice = .false. + cplwav = .false. + cplwav2atm = .false. + do_ca = .false. + ca_global = .false. + ca_sgs = .true. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + nthresh = 18 + ca_trigger = .true. + nspinup = 1 + iseed_ca = 1448371824 +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_version = 1 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 2 + knob_ugwp_ndx4lh = 4 + knob_ugwp_palaunch = 275.0e2 + knob_ugwp_nslope = 1 + knob_ugwp_lzmax = 15.750e3 + knob_ugwp_lzmin = 0.75e3 + knob_ugwp_lzstar = 2.0e3 + knob_ugwp_taumin = 0.25e-3 + knob_ugwp_tauamp = 0.5e-3 + knob_ugwp_lhmet = 200.0e3 + knob_ugwp_orosolv = 'pss-1986' +/ + +&ccpp_suite_sim_nml + suite_sim_file = '' + nprc_sim = 7 + prc_LWRAD_cfg = 0, 0, 1 + prc_SWRAD_cfg = 0, 0, 2 + prc_PBL_cfg = 1, 0, 3 + prc_GWD_cfg = 1, 0, 4 + prc_SCNV_cfg = 1, 1, 5 + prc_DCNV_cfg = 1, 1, 6 + prc_cldMP_cfg = 1, 1, 7 +/ diff --git a/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP_ps.nml b/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP_ps.nml new file mode 100644 index 000000000..c562957f2 --- /dev/null +++ b/ccpp/physics_namelists/input_GFS_v17_HR3_RRTMGP_ps.nml @@ -0,0 +1,171 @@ +&gfs_physics_nml + fhzero = 6 + h2o_phys = .true. + ldiag3d = .true. + qdiag3d = .true. + print_diff_pgr = .false. + fhcyc = 24 + use_ufo = .true. + pre_rad = .false. + imp_physics = 8 + iovr = 3 + ltaerosol = .False. + lradar = .true. + ttendlim = -999 + dt_inner = 150 + sedi_semi = .true. + decfl = 10 + oz_phys = .false. + oz_phys_2015 = .true. + lsoil_lsm = 4 + do_mynnedmf = .false. + do_mynnsfclay = .false. + icloud_bl = 1 + bl_mynn_edmf = 1 + bl_mynn_tkeadvect = .true. + bl_mynn_edmf_mom = 1 + do_ugwp = .false. + do_tofd = .false. + gwd_opt = 2 + do_ugwp_v0 = .false. + do_ugwp_v1 = .true. + do_ugwp_v0_orog_only = .false. + do_ugwp_v0_nst_only = .false. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_ugwp_v1_orog_only = .false. + min_lakeice = 0.15 + min_seaice = 1.0e-6 + use_cice_alb = .true. + pdfcld = .false. + fhswr = 1200. + fhlwr = 1200. + progsigma = .true. + betascu = 8.0 + betamcu = 1.0 + betadcu = 2.0 + ialb = 2 + iems = 2 + iaer = 1011 + icliq_sw = 2 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + shal_cnv = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + satmedmf = .true. + isatmedmf = 1 + lheatstrg = .false. + lseaspray = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + imfshalcnv = 2 + imfdeepcnv = 2 + ras = .false. + cdmbgwd = 2.5,7.5,1.0,1.0 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = 2 + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = 3 + iopt_trs = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + debug = .false. + nstf_name = 2,0,0,0,0 + nst_anl = .true. + psautco = 0.0008,0.0005 + prautco = 0.00015,0.00015 + lgfdlmprad = .false. + effr_in = .true. + ldiag_ugwp = .false. + do_sppt = .false. + do_shum = .false. + do_skeb = .false. + do_RRTMGP = .true. + doGP_cldoptics_LUT = .true. + doGP_lwscat = .true. + active_gases = 'h2o_co2_o3_n2o_ch4_o2' + ngases = 6 + rrtmgp_root = '../../ccpp/physics/physics/Radiation/RRTMGP/rte-rrtmgp/' + lw_file_gas = 'rrtmgp/data/rrtmgp-data-lw-g128-210809.nc' + lw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-lw.nc' + sw_file_gas = 'rrtmgp/data/rrtmgp-data-sw-g112-210809.nc' + sw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-reordered-sw.nc' + rrtmgp_nGptsSW = 112 + rrtmgp_nGptsLW = 128 + rrtmgp_nBandsLW = 16 + rrtmgp_nBandsSW = 14 + frac_grid = .true. + cplchm = .false. + cplflx = .false. + cplice = .false. + cplwav = .false. + cplwav2atm = .false. + do_ca = .false. + ca_global = .false. + ca_sgs = .true. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + nthresh = 18 + ca_trigger = .true. + nspinup = 1 + iseed_ca = 1448371824 +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_version = 1 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 2 + knob_ugwp_ndx4lh = 4 + knob_ugwp_palaunch = 275.0e2 + knob_ugwp_nslope = 1 + knob_ugwp_lzmax = 15.750e3 + knob_ugwp_lzmin = 0.75e3 + knob_ugwp_lzstar = 2.0e3 + knob_ugwp_taumin = 0.25e-3 + knob_ugwp_tauamp = 0.5e-3 + knob_ugwp_lhmet = 200.0e3 + knob_ugwp_orosolv = 'pss-1986' +/ + +&ccpp_suite_sim_nml + suite_sim_file = '' + nprc_sim = 7 + prc_LWRAD_cfg = 0, 0, 1 + prc_SWRAD_cfg = 0, 0, 2 + prc_PBL_cfg = 1, 0, 3 + prc_GWD_cfg = 1, 0, 4 + prc_SCNV_cfg = 1, 1, 5 + prc_DCNV_cfg = 1, 1, 6 + prc_cldMP_cfg = 1, 1, 7 +/ diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP.xml new file mode 100644 index 000000000..09bcbfa64 --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP.xml @@ -0,0 +1,89 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmgp_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmgp_pre + GFS_radiation_surface + GFS_rrtmgp_cloud_mp + GFS_rrtmgp_cloud_overlap + GFS_cloud_diagnostics + rrtmgp_aerosol_optics + rrtmgp_sw_main + rrtmgp_lw_main + GFS_rrtmgp_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP_ps.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP_ps.xml new file mode 100644 index 000000000..0ba77dff3 --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3_RRTMGP_ps.xml @@ -0,0 +1,70 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmgp_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmgp_pre + GFS_radiation_surface + GFS_rrtmgp_cloud_mp + GFS_rrtmgp_cloud_overlap + GFS_cloud_diagnostics + rrtmgp_aerosol_optics + rrtmgp_sw_main + rrtmgp_lw_main + GFS_rrtmgp_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + scm_sfc_flux_spec + dcyc2t3 + GFS_suite_interstitial_2 + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index fb60e12ae..adfe33350 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -45,6 +45,7 @@ def timestep(self, value): suite_list.append(suite('SCM_GFS_v16', 'tracers_GFS_v16.txt', 'input_GFS_v16.nml', 600.0, 1800.0, True )) suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_HR3.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) +suite_list.append(suite('SCM_GFS_v17_HR3_RRTMGP','tracers_GFS_v17_p8.txt', 'input_GFS_v17_HR3_RRTMGP.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) From 6e2c33a8fca006f76ea508c2e7e2745073331af8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 3 Apr 2024 13:42:24 -0600 Subject: [PATCH 156/189] Add a macos clang module file --- scm/etc/modules/macos_clang.lua | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 scm/etc/modules/macos_clang.lua diff --git a/scm/etc/modules/macos_clang.lua b/scm/etc/modules/macos_clang.lua new file mode 100644 index 000000000..71ab2a984 --- /dev/null +++ b/scm/etc/modules/macos_clang.lua @@ -0,0 +1,32 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +a MacOS machine with clang v15.0.0 compilers +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang v15.0.0 compilers]===]) + +local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") +prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") + +load("stack-apple-clang/15.0.0") +load("stack-python/3.10.13") +load("stack-openmpi/4.1.6") + + +load("cmake/3.28.3") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") + +load("py-f90nml/1.4.3") +load("py-netcdf4/1.5.8") + + +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +setenv("CMAKE_C_COMPILER","clang") +setenv("CMAKE_CXX_COMPILER","clang++") +setenv("CMAKE_Fortran_COMPILER","gfortran-12") +setenv("CMAKE_Platform","macos.clang") From 1f6014a6120e4b2ef703b1cb374b6848646ee87e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 5 Apr 2024 09:51:52 -0600 Subject: [PATCH 157/189] Starting documentation updates for build/run - Update Quick Start up through "Compilers" section - Update Repository chapter to include new "Testing" section (needs more expansion later) - Convert section marks to conform with Sphinx Documentation - Fix link and syntax errors --- scm/doc/TechGuide/chap_intro.rst | 2 +- scm/doc/TechGuide/chap_quick.rst | 89 ++++++++++++++------------------ scm/doc/TechGuide/chap_repo.rst | 21 ++++++++ 3 files changed, 60 insertions(+), 52 deletions(-) diff --git a/scm/doc/TechGuide/chap_intro.rst b/scm/doc/TechGuide/chap_intro.rst index 68e4ddf1d..94d134d85 100644 --- a/scm/doc/TechGuide/chap_intro.rst +++ b/scm/doc/TechGuide/chap_intro.rst @@ -103,7 +103,7 @@ This release bundle has some known limitations: - As of this release, using the SCM over a land point with an LSM is possible through the use of UFS initial conditions (see - :numref:`Section %s `). However, advective forcing terms + :numref:`Section %s `). However, advective forcing terms are unavailable as of this release, so only short integrations using this configuration should be employed. Using dynamical tendencies (advective forcing terms) from the UFS will be part of a future diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 69f165b94..e5554b810 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -29,7 +29,7 @@ developer code, but may not be as stable or consistent with existing documentati Instructions for using either option are discussed here. Release Code -~~~~~~~~~~~~ +^^^^^^^^^^^^ Clone the source using @@ -54,7 +54,7 @@ this level. The CCPP physics parameterizations can be found in the .. _`development_code`: Development Code -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ If you would like to contribute as a developer to this project, please see (in addition to the rest of this guide) the scientific and technical @@ -133,62 +133,49 @@ relies on the NetCDF libraries. Beyond the standard scripts, the build system relies on use of the Python scripting language, along with cmake, GNU make and date. -The following software stacks have been tested with this code. Other -versions of various components will likely still work, however. -- gfortran 12.1.0, gcc 12.1.0, cmake 3.23.2, NetCDF 4.7.4, Python - 3.9.12 +For the latest release, the minimum required Python version is 3.8, and CMake requires a minimum version of 3.14. +While exact minimum required versions of other prerequisites have not been established, users can reference the +list of Continuous Integration tests run on the CCPP SCM repository (see :numref:`Section %s `) +for examples of known working configurations. -- GNU compilers 10.1.0, cmake 3.16.4, NetCDF 4.8.1, Python 3.7.12 +Spack-stack +^^^^^^^^^^^^ -- GNU compilers 11.1.0, cmake 3.18.2, NetCDF 4.8.1, Python 3.8.5 +A joint effort between NOAA's Unified Forecast System (UFS) and Joint Effort for Data assimilation Integration (JEDI). +It is designed to be a comprehensive, all-in-one package containing prerequisite libraries and tools needed for all +software in the UFS ecosystem, including the CCPP SCM. As of the version 7, installing spack-stack is the main +supported method of installing the prerequisites needed for building the SCM. The latest version of the SCM is meant +to be built with spack-stack v1.6.0. Older versions may work, but are not guaranteed. Version 1.6.0 of spack-stack +contains the following set of libraries needed for building the SCM: -- Intel compilers 2022.0.2, cmake 3.20.1, NetCDF 4.7.4, Python 3.7.11 + - Netcdf-c (v4.9.2) -- Intel compilers 2022.1.0, cmake 3.22.0, NetCDF 4.8.1, Python 3.7.12 + - Netcdf-FORTRAN (v4.6.0) + + - BACIO (v2.4.1) - Binary I/O Library + + - SP (v2.3.3) - Spectral Transformation Library + + - W3EMC (2.10.0) - GRIB decoder and encoder library + +Instructions for installing spack-stack can be found in the `spack-stack documentation `__. +Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and +Jet, and MSU's Orion and Hercules. Because these tools are typically the purview of system administrators to install and maintain, they are considered part of the basic system -requirements. The Unified Forecast System (UFS) Short-Range Weather -Application release v1.0.0 of March 2021 provides software packages and -detailed instructions to install these prerequisites and the hpc-stack -on supported platforms (see :numref:`Section %s `) - -Further, there are several utility libraries as part of the hpc-stack -package that must be installed with environment variables pointing to -their locations prior to building the SCM. - -- bacio - Binary I/O Library - -- sp - Spectral Transformation Library - -- w3emc - GRIB decoder and encoder library - -The following environment variables are used by the build system to -properly link these libraries: ``bacio_ROOT``, ``sp_ROOT``, and ``w3emc_ROOT`` Computational platforms on -which these libraries are prebuilt and installed in a central location -are referred to as *preconfigured* platforms. Examples of preconfigured -platforms are most NOAA high-performance computing machines (using the -Intel compiler) and the NCAR Cheyenne system (using the Intel and GNU -compilers). The machine setup scripts mentioned in -:numref:`Section %s ` load these libraries (which are identical -to those used by the UFS Short and Medium Range Weather Applications on -those machines) and set these environment variables for the user -automatically. For installing the libraries and its prerequisites on -supported platforms, existing UFS packages can be used (see -:numref:`Section %s `). +requirements. Compilers -~~~~~~~~~ +^^^^^^^^^ The CCPP and SCM have been tested on a variety of computing platforms. Currently the CCPP system is actively supported on Linux and MacOS computing platforms using the Intel or GNU Fortran compilers. Windows users have a path to use this software through a Docker container that -uses Linux internally (see section `1.5 <#docker>`__). Please use -compiler versions listed in the previous section as unforeseen build -issues may occur when using older versions. Typically the best results -come from using the most recent version of a compiler. If you have +uses Linux internally (see :numref:`Section %s `). Typically the best chance of successfully building and +running the SCM on a new machine comes from using the most recent version of a compiler. If you have problems with compilers, please check the “Known Issues” section of the release website (https://dtcenter.org/community-code/common-community-physics-package-ccpp/download). @@ -196,7 +183,7 @@ release website .. _`use_preconfigured_platforms`: Using Existing Libraries on Preconfigured Platforms -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Because the SCM can be built using the so-called `"spack-stack" libraries `__ @@ -223,7 +210,7 @@ both building and running the SCM. .. _`setup_supported_platforms`: Installing Libraries on Non-preconfigured Platforms -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, the project is suggested for @@ -442,7 +429,7 @@ integrations serially, depending on the command line arguments supplied. .. _`singlerunscript`: Run Script Usage -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ Running a case requires four pieces of information: the case to run (consisting of initial conditions, geolocation, forcing data, etc.), the @@ -518,7 +505,7 @@ below where option abbreviations are included in brackets. value. This is used, for example, in regression testing to reduce total runtimes. -- ``--levels [-l] +- ``--levels [-l]`` - Use this to change the number of vertical levels. @@ -667,7 +654,7 @@ volume-mounting purposes. Any standard NetCDF file viewing or analysis tools may be used to examine the output file (ncdump, ncview, NCL, etc). Batch Run Script -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ If using the model on HPC resources and significant amounts of processor time is anticipated for the experiments, it will likely be necessary to @@ -728,7 +715,7 @@ internet search. docker-machine create default --virtualbox-no-vtx-check Building the Docker image -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ The Dockerfile builds CCPP SCM v6.0.0 from source using the GNU compiler. A number of required codes are built and installed via the @@ -787,7 +774,7 @@ that was installed as part of Docker Toolbox. docker images Using a prebuilt Docker image from Dockerhub -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A prebuilt Docker image for this release is available on Dockerhub if it is not desired to build from source. In order to use this, execute the @@ -804,7 +791,7 @@ To verify that it exists afterward, run docker images Running the Docker image -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ NOTE: Windows users can execute these steps through the Docker Quickstart application installed with Docker Toolbox. diff --git a/scm/doc/TechGuide/chap_repo.rst b/scm/doc/TechGuide/chap_repo.rst index d49b9605a..36f0a10ce 100644 --- a/scm/doc/TechGuide/chap_repo.rst +++ b/scm/doc/TechGuide/chap_repo.rst @@ -60,3 +60,24 @@ Cubed-Sphere (FV3) dynamical core. | ``│   └── src`` - source code for SCM infrastructure, Python run script, CMakeLists.txt for the SCM, example multirun setup files, suite_info.py | ``└── test`` - Contains scripts for regression testing, Continuous Integration tests +Testing +----------------- + +Regression Testing +^^^^^^^^^^^^^^^^^^ + +Regression tests are a comprehensive set of build and run tests meant to ensure that new changes to the SCM do not break any existing capabilities. These tests are run on code changes before they are merged, and so ensure that the ``main`` branch is always free of major bugs in all facets of the system covered by the tests. + +The latest set of Regression tests are run automatically for every new code change when a Pull Request is opened via GitHub's `Continuous Integration`_. Regression tests are also run manually on a wide variety of platforms in preparation for code release to ensure that all capabilities work as expected for a reasonable spectrum of possible machines a user might want to use. + +Continuous Integration +^^^^^^^^^^^^^^^^^^^^^^ + +The CCPP SCM GitHub repository is set up with Continuous Integration (CI) testing for building the SCM and running some simple test cases. These tests are run automatically on code changes before they are merged, and so ensures that new changes to the SCM do not break basic capabilities. The latest set of tests use the following combinations of SCM prerequisites: + +**Regression tests** + - GNU compilers 11.4.0, Python 3.9.12, netCDF-c 4.7.3, netCDF-FORTRAN 4.5.3, bacio 2.4.1, sp 2.3.3, and w3emc 2.9.2 + +**Build tests** + +All tests use the same versions of NCEP-supported libraries: bacio 2.4.1, sp 2.3.3, and w3emc 2.9.2. Detailed information on these tests can be found in the definition files for these tests, stored in the SCM repository under ``ccpp-scm/.github/workflows``. From 8d90db0fab8438c251bc72b10cb7c146337028c6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Thu, 25 Apr 2024 13:56:14 -0600 Subject: [PATCH 158/189] Update dockerfile for latest main branch --- docker/Dockerfile | 108 ++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 99fc2a25b..bc78ad72a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,52 +1,88 @@ -FROM dtcenter/common-community-container:gnu9 +FROM debian:12 +MAINTAINER Michael Kavulich -MAINTAINER Michelle Harrold or Grant Firl or Michael Kavulich +# Set up base OS environment -# -# Dockerfile for building CCPP SCM container -# +RUN apt-get -y update -# Note: The common community container image contains the following packages, which are used to build the SCM: -# gfortran, gcc, cmake, netCDF, HDF5, ZLIB, SZIP, Python, and libxml2 -# To access the common community container repository: https://github.com/NCAR/Common-Community-Container +# Get "essential" tools and libraries +RUN apt-get -y install build-essential +RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim +RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 + +MAINTAINER Grant Firl or Michael Kavulich + +#Compiler environment variables +ENV CC /usr/bin/gcc +ENV FC /usr/bin/gfortran +ENV CXX /usr/bin/g++ +ENV F77 /usr/bin/gfortran +ENV F90 /usr/bin/gfortran + +# Other necessary environment variables +ENV LD_LIBRARY_PATH /usr/lib/ + +# Set up unpriviledged user account +RUN groupadd comusers -g 9999 +RUN useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser +RUN mkdir /comsoftware \ + && chown -R comuser:comusers /comsoftware \ + && chmod 6755 /comsoftware + +# Set up user home space correctly and make sure user has permissions on all stuff in /comsoftware +RUN chown -R comuser:comusers /home \ + && chmod 6755 /home +RUN chown -R comuser:comusers /comsoftware \ + && chmod -R 6755 /comsoftware + +# Link version-specific aliases +RUN ln -s /usr/bin/python3 /usr/local/bin/python +RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran + +# all root steps completed above, now continue below as regular userID comuser +USER comuser +WORKDIR /home + +# Build NCEP libraries we need for SCM + +ENV NCEPLIBS_DIR /comsoftware/nceplibs + +RUN mkdir -p $NCEPLIBS_DIR/src && cd $NCEPLIBS_DIR/src \ + && git clone -b v2.4.1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-bacio \ + && mkdir NCEPLIBS-bacio/build && cd NCEPLIBS-bacio/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install + +RUN cd $NCEPLIBS_DIR/src \ + && git clone -b v2.3.3 --recursive https://github.com/NOAA-EMC/NCEPLIBS-sp \ + && mkdir NCEPLIBS-sp/build && cd NCEPLIBS-sp/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install + +RUN cd $NCEPLIBS_DIR/src \ + && git clone -b v2.11.0 --recursive https://github.com/NOAA-EMC/NCEPLIBS-w3emc \ + && mkdir NCEPLIBS-w3emc/build && cd NCEPLIBS-w3emc/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install # Obtain CCPP SCM source code RUN cd /comsoftware \ - && git clone --recursive -b release/public-v6 https://github.com/NCAR/ccpp-scm + && git clone --recursive -b main https://github.com/NCAR/ccpp-scm # Obtain static data that was previously stored in repository RUN cd /comsoftware/ccpp-scm/ \ - && . contrib/get_all_static_data.sh + && ./contrib/get_all_static_data.sh # Obtain the pre-computed look-up tables for running with Thompson microphysics RUN cd /comsoftware/ccpp-scm/ \ - && . contrib/get_thompson_tables.sh + && ./contrib/get_thompson_tables.sh -# Run the machine setup script to set environment variables -ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc -ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/g++ -ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran -ENV F90=/opt/rh/devtoolset-9/root/usr/bin/gfortran -ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran - -ENV NETCDF=/comsoftware/libs/netcdf - -RUN cd /comsoftware/ccpp-scm/contrib \ - && wget https://raw.githubusercontent.com/NCAR/ccpp-scm/3f501aa8af0fb00ff124d8301c932292d1d0abf3/contrib/build_nceplibs.sh \ - && chmod +x build_nceplibs.sh \ - && cd .. \ - && ./contrib/build_nceplibs.sh $PWD/nceplibs - -ENV bacio_ROOT /comsoftware/ccpp-scm/nceplibs -ENV sp_ROOT /comsoftware/ccpp-scm/nceplibs -ENV w3nco_ROOT /comsoftware/ccpp-scm/nceplibs - -# Create your own link from python -> python3 -# This works without setting the system PATH env var -# since /usr/local/bin is before /usr/bin in the search path. -USER root -RUN ln -s /usr/bin/python3 /usr/local/bin/python -USER comuser +ENV bacio_ROOT /comsoftware/nceplibs +ENV sp_ROOT /comsoftware/nceplibs +ENV w3emc_ROOT /comsoftware/nceplibs # Invoke cmake on the source code to build RUN cd /comsoftware/ccpp-scm/scm \ From e0e18f7f0fd07dfa159d69143423037ceb2bfdf9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 29 Apr 2024 09:46:19 -0600 Subject: [PATCH 159/189] Update Dockerfile python setup; now runs test case successfully. Update docs for latest python packages --- docker/Dockerfile | 14 ++++++++++++-- scm/doc/TechGuide/chap_quick.rst | 32 +++++++++----------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bc78ad72a..721dda5d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get -y update RUN apt-get -y install build-essential RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 +RUN apt-get -y install python3-pip python3.11-venv MAINTAINER Grant Firl or Michael Kavulich @@ -35,8 +36,8 @@ RUN chown -R comuser:comusers /home \ RUN chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware -# Link version-specific aliases -RUN ln -s /usr/bin/python3 /usr/local/bin/python +# Link version-specific aliases (python3 will be created later with virtual environment) +RUN ln -s ~comuser/.venv/bin/python3 /usr/local/bin/python RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran # all root steps completed above, now continue below as regular userID comuser @@ -103,3 +104,12 @@ RUN cd /comsoftware/ccpp-scm/scm \ WORKDIR /comsoftware/ccpp-scm/scm/bin ENV SCM_WORK=/comsoftware ENV SCM_ROOT=/comsoftware/ccpp-scm/ + +# For interactive use, vim mouse settings are infuriating +RUN echo "set mouse=" > ~/.vimrc + +# Set up python virtual environment and install needed packages +ENV VIRTUAL_ENV=~/.venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install f90nml==1.4.4 netcdf4==1.6.5 diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index e5554b810..1093263d9 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -169,7 +169,6 @@ requirements. Compilers ^^^^^^^^^ - The CCPP and SCM have been tested on a variety of computing platforms. Currently the CCPP system is actively supported on Linux and MacOS computing platforms using the Intel or GNU Fortran compilers. Windows @@ -227,44 +226,31 @@ prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use UFS can install only NetCDF/NetCDF-Fortran manually or using the software package manager (apt, yum, brew). -The Python environment must provide the module for the SCM scripts to -function. Users can test if f90nml is installed using this command in +The Python environment must provide a few non-default modules for the SCM scripts to +function: ``f90nml`` (`documentation `__) and +``netcdf4`` (`documentation `__). Users can test if these are installed using this command in the shell: :: - python -c "import f90nml" + python -c "import f90nml; import netcdf4" If is installed, this command will succeed silently, otherwise an ``ImportError: No module named f90nml`` -will be printed to screen. To install the ``f90nml`` (v0.19) Python module, use the -install method preferred for your Python environment (one of the -following): +will be printed to screen. To install the ``f90nml`` (v1.4.4; ) and ``netcdf4`` (v1.6.5) Python modules, use the +install method preferred for your Python environment (one of the following): - :: - easy_install f90nml==0.19 + easy_install f90nml==1.4.4 netcdf4==1.6.5 - :: - pip install f90nml==0.19 + pip install f90nml==1.4.4 netcdf4==1.6.5 - :: - conda install f90nml=0.19 - -or perform the following steps to install it manually from source: - -:: - - cd /directory/with/write/priveleges - git clone -b v0.19 https://github.com/marshallward/f90nml - cd f90nml - python setup.py install [--prefix=/my/install/directory or --user] + conda install -c conda-forge f90nml==1.4.4 netcdf4==1.6.5 -The directory ``/my/install/directory`` must exist and its subdirectory -``/my/install/directory/lib/python[version]/site-packages`` (or ``lib64`` -instead of ``lib``, depending on the system) must be in the ``PYTHONPATH`` -environment variable. .. _`compiling`: From d1d16a7373916e9c2d2e7d71f6628854c2104cd7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 09:44:18 -0600 Subject: [PATCH 160/189] Testing out docker autobuild CI --- .github/workflows/ci_docker.yml | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/ci_docker.yml diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml new file mode 100644 index 000000000..0aedbde24 --- /dev/null +++ b/.github/workflows/ci_docker.yml @@ -0,0 +1,42 @@ +name: build_test_and_push_docker + +on: + push: +# branches: +# - "main" + +env: + TEST_TAG: mkavulich/ccpp-scm:test + LATEST_TAG: mkavulich/ccpp-scm:latest + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ env.TEST_TAG }} + - name: Test + run: | + docker run --rm -it -v ${OUT_DIR}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.LATEST_TAG }} From b6a1c3167551d1c3f079ad92fcc30c9ba9ea83c7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 09:56:46 -0600 Subject: [PATCH 161/189] Don't run CI on all pushes --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9e998655f..8bb7fcedb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: From 5c9b124e5242c807ab52906b86c7f778cc5c27b9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:02:40 -0600 Subject: [PATCH 162/189] Fixing docker context for CI --- .github/workflows/ci_docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0aedbde24..cc0634267 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -28,15 +28,17 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: docker/Dockerfile load: true tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v ${OUT_DIR}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -it -v ${{ env.HOME }}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: context: . + file: docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ env.LATEST_TAG }} From 6b12f31d7c534ec988d590865799ec092b7da65e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:23:42 -0600 Subject: [PATCH 163/189] Small users guide addition about Docker usage, try to fix docker CI home space specification --- .github/workflows/ci_docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index cc0634267..2a38887be 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,7 +24,7 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export to Docker + - name: Build and export test tag to Docker uses: docker/build-push-action@v5 with: context: . @@ -33,7 +33,7 @@ jobs: tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v ${{ env.HOME }}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -it -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From cc88611fa939fd09349a3efb99f5815b8c217dcd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:44:28 -0600 Subject: [PATCH 164/189] Testing docker CI --- .github/workflows/ci_docker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 2a38887be..cdff25f53 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,16 +24,16 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - load: true - tags: ${{ env.TEST_TAG }} +# - name: Build and export test tag to Docker +# uses: docker/build-push-action@v5 +# with: +# context: . +# file: docker/Dockerfile +# load: true +# tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From edf6b18d2b56777c7ef4614193a195f28c0de9b5 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:54:34 -0600 Subject: [PATCH 165/189] Testing push of the test tag to dockerhub --- .github/workflows/ci_docker.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index cdff25f53..a4b904785 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,13 +24,14 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} -# - name: Build and export test tag to Docker -# uses: docker/build-push-action@v5 -# with: -# context: . -# file: docker/Dockerfile -# load: true -# tags: ${{ env.TEST_TAG }} + - name: Build and export test tag to Docker + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + load: true + tags: ${{ env.TEST_TAG }} + push: true - name: Test run: | docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d From f12b0b85e497e54602df218d6d1b7ffae02faf97 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:21:50 -0600 Subject: [PATCH 166/189] Testing pull of docker CI test tag --- .github/workflows/ci_docker.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index a4b904785..35684fe03 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,16 +24,17 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - load: true - tags: ${{ env.TEST_TAG }} - push: true +# - name: Build and export test tag to Docker +# uses: docker/build-push-action@v5 +# with: +# context: . +# file: docker/Dockerfile +# load: true +# tags: ${{ env.TEST_TAG }} +# push: true - name: Test run: | + docker pull ${{ env.TEST_TAG }} docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 From 048b01888ef87cfb5196ae1f8bffa93196342ce4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:41:44 -0600 Subject: [PATCH 167/189] Testing HOME permissions for docker CI --- .github/workflows/ci_docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 35684fe03..0f63de472 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -35,6 +35,8 @@ jobs: - name: Test run: | docker pull ${{ env.TEST_TAG }} + echo $HOME + ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 From d56352453966428936459c51ac688badf8a429e2 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:45:19 -0600 Subject: [PATCH 168/189] Adding write permissions to $HOME for docker CI --- .github/workflows/ci_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0f63de472..91bf3f493 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,6 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} echo $HOME + chmod a+w $HOME ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 9c8556f395bfcf2cbb19f4ffa3e66cb6045d61bd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:47:50 -0600 Subject: [PATCH 169/189] Adding read+write permissions to $HOME recursively wqfor docker CI --- .github/workflows/ci_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 91bf3f493..b4336a672 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} echo $HOME - chmod a+w $HOME + chmod +rw -R $HOME ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 2d0182e1be79dd19b776bc1e024600dd8ffbdcf8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:50:30 -0600 Subject: [PATCH 170/189] Put output in a subdirectory of $HOME to avoid permissions issues for docker CI --- .github/workflows/ci_docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index b4336a672..0530fb5cd 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -35,10 +35,10 @@ jobs: - name: Test run: | docker pull ${{ env.TEST_TAG }} - echo $HOME - chmod +rw -R $HOME - ls -al $HOME - docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + mkdir $HOME/output + chmod +rw -R $HOME/output + ls -al $HOME/output + docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From a0c0c9f7470108ea7d79e770b79917b7ccb72709 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:58:39 -0600 Subject: [PATCH 171/189] chmod wasn't working as expected --- .github/workflows/ci_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0530fb5cd..928159a00 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +rw -R $HOME/output + chmod +r +w $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 4f5d74d446d12b353b96edb901de2a1ed6b1b40e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:00:32 -0600 Subject: [PATCH 172/189] chmod STILL wasn't working as expected --- .github/workflows/ci_docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 928159a00..eb9ac5e75 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,8 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +r +w $HOME/output + chmod +r $HOME/output + chmod +w $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From a9bda64b0a5fa377c2270d4b45ef4c1e1801b834 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:04:25 -0600 Subject: [PATCH 173/189] I will never ever trust myself with a chmod command again --- .github/workflows/ci_docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index eb9ac5e75..d809ce2aa 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,8 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +r $HOME/output - chmod +w $HOME/output + chmod a+rw $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 4087a9fb33573ef4eeab127ca0ed339bc35a9d7a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:10:41 -0600 Subject: [PATCH 174/189] It works! Now to try running all test cases --- .github/workflows/ci_docker.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index d809ce2aa..8e2be6ee2 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -32,13 +32,12 @@ jobs: # load: true # tags: ${{ env.TEST_TAG }} # push: true - - name: Test + - name: Test and push latest tag run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output - ls -al $HOME/output - docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - name: Build and push uses: docker/build-push-action@v5 with: From 19350a59fe58c7ac51a0fd447015aa772de3fd00 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:14:08 -0600 Subject: [PATCH 175/189] Now to run it all together... --- .github/workflows/ci_docker.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 8e2be6ee2..1dd664b52 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,14 +24,13 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} -# - name: Build and export test tag to Docker -# uses: docker/build-push-action@v5 -# with: -# context: . -# file: docker/Dockerfile -# load: true -# tags: ${{ env.TEST_TAG }} -# push: true + - name: Build and export test tag to Docker + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + load: true + tags: ${{ env.TEST_TAG }} - name: Test and push latest tag run: | docker pull ${{ env.TEST_TAG }} From e1ce0ccfbfcc8563e62c30bbce913ea2b31e66d3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:21:41 -0600 Subject: [PATCH 176/189] Consolidate dockerfile RUN commands to shrink container size --- .github/workflows/ci_docker.yml | 6 +++--- docker/Dockerfile | 29 +++++++++++------------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 1dd664b52..65e07d85e 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,20 +24,20 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker + - name: Build and export test tag uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile load: true tags: ${{ env.TEST_TAG }} - - name: Test and push latest tag + - name: Test run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - - name: Build and push + - name: Push latest tag uses: docker/build-push-action@v5 with: context: . diff --git a/docker/Dockerfile b/docker/Dockerfile index 721dda5d0..9699254c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,10 +6,10 @@ MAINTAINER Michael Kavulich RUN apt-get -y update # Get "essential" tools and libraries -RUN apt-get -y install build-essential -RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim -RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 -RUN apt-get -y install python3-pip python3.11-venv +RUN apt-get -y install build-essential \ + && apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim \ + && apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 \ + && apt-get -y install python3-pip python3.11-venv MAINTAINER Grant Firl or Michael Kavulich @@ -69,24 +69,17 @@ RUN cd $NCEPLIBS_DIR/src \ && make VERBOSE=1 \ && make install -# Obtain CCPP SCM source code -RUN cd /comsoftware \ - && git clone --recursive -b main https://github.com/NCAR/ccpp-scm - -# Obtain static data that was previously stored in repository -RUN cd /comsoftware/ccpp-scm/ \ - && ./contrib/get_all_static_data.sh - -# Obtain the pre-computed look-up tables for running with Thompson microphysics -RUN cd /comsoftware/ccpp-scm/ \ - && ./contrib/get_thompson_tables.sh - ENV bacio_ROOT /comsoftware/nceplibs ENV sp_ROOT /comsoftware/nceplibs ENV w3emc_ROOT /comsoftware/nceplibs -# Invoke cmake on the source code to build -RUN cd /comsoftware/ccpp-scm/scm \ +# Obtain CCPP SCM source code and static data, build code +RUN cd /comsoftware \ + && git clone --recursive -b main https://github.com/NCAR/ccpp-scm \ + && cd /comsoftware/ccpp-scm/ \ + && ./contrib/get_all_static_data.sh \ + && ./contrib/get_thompson_tables.sh \ + && cd /comsoftware/ccpp-scm/scm \ && mkdir bin \ && cd bin \ && cmake ../src \ From 78ad73c510bf153e4a68b9bb5e5133c16e01fe98 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 20:45:03 -0600 Subject: [PATCH 177/189] More updates to Quick Start chapter: - remove a lot of fluff to get to the details quicker - Add example commands for installing spack-stack on MacOS - Update "multi-run" command for Docker Now updated through "4.3. Compiling SCM with CCPP" --- .../{ci_docker.yml => ci_test_docker.yml} | 0 scm/doc/TechGuide/chap_quick.rst | 129 ++++++++++++------ 2 files changed, 84 insertions(+), 45 deletions(-) rename .github/workflows/{ci_docker.yml => ci_test_docker.yml} (100%) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_test_docker.yml similarity index 100% rename from .github/workflows/ci_docker.yml rename to .github/workflows/ci_test_docker.yml diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 1093263d9..c54ff8f8c 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -4,18 +4,8 @@ Quick Start Guide ================= This chapter provides instructions for obtaining and compiling the CCPP -SCM. The SCM code calls CCPP-compliant physics schemes through the CCPP -framework code. As such, it requires the CCPP framework code and physics -code, both of which are included as submodules within the SCM git repository. This -package can be considered a simple example for an atmospheric model to -interact with physics through the CCPP. - -Alternatively, if one doesn’t have access to or care to set up a machine -with the appropriate system requirements but has a working Docker -installation, it is possible to create and use a Docker container with a -pre-configured computing environment with a pre-compiled model. This is -also an avenue for running this software with a Windows PC. See section -:numref:`Section %s ` for more information. +SCM. We provide instructions on building the code from scratch (:numref:`Section %s `), as well as +using Docker containers for machines that have Docker software installed (:numref:`Section %s `). .. _obtaining_code: @@ -37,11 +27,10 @@ Clone the source using git clone --recursive -b v6.0.0 https://github.com/NCAR/ccpp-scm -By using the ``--recursive`` option, it guarantees that you are checking out the commits -of ccpp-physics and ccpp-framework that were tested with the latest -commit of the SCM main branch. If not included initially, you can always retrieve the commits of -the submodules that were intended to be used with a given commit of the -SCM by executing the following command from the SCM directory: +The ``--recursive`` option is required to retrieve the ccpp-physics and ccpp-framework code, +which are stored in separate repositories and linked to the SCM repository as submodules. +If not included initially, you can always retrieve the submodules +by executing the following command from the SCM directory: .. code:: bash @@ -56,17 +45,8 @@ this level. The CCPP physics parameterizations can be found in the Development Code ^^^^^^^^^^^^^^^^ -If you would like to contribute as a developer to this project, please -see (in addition to the rest of this guide) the scientific and technical -documentation included with this release for both the SCM and the CCPP: - -https://dtcenter.org/community-code/common-community-physics-package-ccpp/documentation - -There you will find links to all of the documentation pertinent to -developers. - -For working with the development branches (stability not guaranteed), -check out the ``main`` branch of the repository: +Developers seeking to contribute code to the SCM or CCPP will need to use the most up-to-date +version of the code, which can be found on the ``main`` branch of the repository: .. code:: bash @@ -82,7 +62,21 @@ SCM by executing the following command from the SCM directory: git submodule update --init --recursive -You can try to use the latest commits of the ccpp-physics and +While the ``main`` branch is tested regularly for compilation and basic functionality (as described in :numref:`Section %s `), +it may not be as stable or scientifically vetted as the latest release code, and may be lacking in up-to-date documentation. + +If you would like to contribute as a developer to this project, please +see (in addition to the rest of this guide) the scientific and technical +documentation included with this release for both the SCM and the CCPP: + +https://dtcenter.org/community-code/common-community-physics-package-ccpp/documentation + +There you will find links to all of the documentation pertinent to +developers. + + +While the SCM is updated with the latest commits to the CCPP submodules (ccpp-physics and ccpp-framework) +on a fairly regular basis, it may be weeks or even months behind at times. You can try to use the latest commits of the ccpp-physics and ccpp-framework submodules if you wish, but this may not have been tested (i.e. SCM development may lag ccpp-physics and/or ccpp-framework development). To do so: @@ -129,11 +123,12 @@ System Requirements, Libraries, and Tools The source code for the SCM and CCPP components is in the form of programs written in FORTRAN 90 (with some required features from the FORTRAN 2008 standard), and C. In addition, the model I/O -relies on the NetCDF libraries. Beyond the standard scripts, the build +relies on the NetCDF libraries, as well as the NCEP libraries ``bacio``, ``sp`` and ``w3emc``. + +Beyond the standard shell scripts, the build system relies on use of the Python scripting language, along with cmake, GNU make and date. - For the latest release, the minimum required Python version is 3.8, and CMake requires a minimum version of 3.14. While exact minimum required versions of other prerequisites have not been established, users can reference the list of Continuous Integration tests run on the CCPP SCM repository (see :numref:`Section %s `) @@ -163,10 +158,6 @@ Instructions for installing spack-stack can be found in the `spack-stack documen Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and Jet, and MSU's Orion and Hercules. -Because these tools are typically the purview of system administrators -to install and maintain, they are considered part of the basic system -requirements. - Compilers ^^^^^^^^^ The CCPP and SCM have been tested on a variety of computing platforms. @@ -212,21 +203,69 @@ Installing Libraries on Non-preconfigured Platforms ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For users on supported platforms such as generic Linux or macOS systems -that have not been preconfigured, the project is suggested for -installing prerequisite libraries. Visit -https://github.com/NOAA-EMC/hpc-stack for instructions for installing -prerequisite libraries via *hpc-stack* in their docs directory. UFS users who -already installed libraries via the *hpc-stack* package only need to set the -compiler (``CC``, ``CXX``, ``FC``), NetCDF (``NetCDF_ROOT``), and ``bacio``, -``sp`` and ``w3emc`` (``bacio_ROOT``, ``sp_ROOT``, ``w3emc_ROOT``) environment variables to point -to their installation paths in order to compile the SCM. +that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) +is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. + + +MacOS example +"""""""""""""" + +As an example of the steps needed for installing spack-stack on a MacOS machine, here is the list of steps that +resulted in a successful build and run of the SCM on a MacBook Pro with the Ventura 13.6.3 operating system, +using ``clang`` compilers. The necessary prerequisites for spack-stack (see +`spack-stack documentation `__) +were installed using Homebrew. + +:: + + cd your_experiment_dir + export TOP_DIR=`pwd` + mkdir spack_stack_1.6.0 + cd spack_stack_1.6.0 + git clone --recurse-submodules https://github.com/jcsda/spack-stack.git -b 1.6.0 + cd spack-stack + source setup.sh + ulimit -S -s unlimited + spack stack create env --site macos.default --template ufs-weather-model --name SCM + cd envs/SCM + spack env activate -p . + export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" + spack external find --scope system --exclude bison --exclude openssl --exclude python + spack external find --scope system libiconv + spack external find --scope system perl + spack external find --scope system wget + PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" spack external find --scope system curl + PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" spack external find --scope system qt + spack config --scope system add packages:pkg-config:buildable:false + spack compiler find --scope system + unset SPACK_SYSTEM_CONFIG_PATH + spack config add "packages:all:compiler:[apple-clang@15.0.0]" + spack config add "packages:all:providers:mpi:[openmpi@4.1.6]" + spack concretize 2>&1 | tee log.concretize + ../../util/show_duplicate_packages.py -d log.concretize + spack install --verbose --fail-fast 2>&1 | tee log.install + spack module lmod refresh + spack stack setup-meta-modules + + +Setting up compilation environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables +needed for specifying the location of the various prerequisites. Users will need to set variables for the +compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). The SCM uses only a small part of the UFS *hpc-stack* package and has fewer prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use the UFS can install only NetCDF/NetCDF-Fortran manually or using the software package manager (apt, yum, brew). -The Python environment must provide a few non-default modules for the SCM scripts to +Python requirements +""""""""""""""""""""" + +The SCM build system invokes the ``ccpp_prebuild.py`` script, and so the Python environment must be set up prior to building. +As mentioned earlier, a minimum Python version of 3.8 is required. Additionally, there are a few non-default modules required for the SCM to function: ``f90nml`` (`documentation `__) and ``netcdf4`` (`documentation `__). Users can test if these are installed using this command in the shell: @@ -837,7 +876,7 @@ Quickstart application installed with Docker Toolbox. .. code:: bash - docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -m -d + docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d The options included in the above ``run`` commands are the following: From 1ff40ce2da77749e70b175ad0d11a125bdce4374 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 20:48:11 -0600 Subject: [PATCH 178/189] Split out the CI test that pushes new container to Dockerhub, only run this on pushes to main --- .../build_and_push_docker_latest.yml | 34 +++++++++++++++++++ .github/workflows/ci_test_docker.yml | 17 ++-------- 2 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/build_and_push_docker_latest.yml diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml new file mode 100644 index 000000000..d9dd73192 --- /dev/null +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -0,0 +1,34 @@ +name: build_test_and_push_docker + +on: + push: + branches: + # Only build containers when pushing to main + - "main" + +env: + LATEST_TAG: NCAR/ccpp-scm:latest + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push latest tag + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.LATEST_TAG }} diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index 65e07d85e..1fac0d699 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -1,13 +1,9 @@ name: build_test_and_push_docker -on: - push: -# branches: -# - "main" +on: [pull_request,workflow_dispatch] env: - TEST_TAG: mkavulich/ccpp-scm:test - LATEST_TAG: mkavulich/ccpp-scm:latest + TEST_TAG: NCAR/ccpp-scm:test jobs: docker: @@ -33,15 +29,6 @@ jobs: tags: ${{ env.TEST_TAG }} - name: Test run: | - docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - - name: Push latest tag - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ env.LATEST_TAG }} From 3d2ce1bf067cee32dbc3b09ffcd858bf43036bde Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 10:56:57 -0600 Subject: [PATCH 179/189] Add Linux GNU modulefile, remove compiler version from macos_clang modulefile for flexibility --- scm/etc/modules/linux_gnu.lua | 33 +++++++++++++++++++++++++++++++++ scm/etc/modules/macos_clang.lua | 6 +++--- 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 scm/etc/modules/linux_gnu.lua diff --git a/scm/etc/modules/linux_gnu.lua b/scm/etc/modules/linux_gnu.lua new file mode 100644 index 000000000..1c0c65efa --- /dev/null +++ b/scm/etc/modules/linux_gnu.lua @@ -0,0 +1,33 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +a generic Linux machine with GNU compilers. Note that you may have to make +modifications to this file based on your own compilers and specifics of +how spack-stack was installed; see the Users Guide for details +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on a Linux machine with GNU compilers]===]) + +local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") +prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") + +load("stack-gnu") +load("stack-python/3.10.13") +load("stack-openmpi/4.1.6") + + +load("cmake/3.28.3") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") + +load("py-f90nml/1.4.3") +load("py-netcdf4/1.5.8") + + +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/scm/etc/modules/macos_clang.lua b/scm/etc/modules/macos_clang.lua index 71ab2a984..41aef7680 100644 --- a/scm/etc/modules/macos_clang.lua +++ b/scm/etc/modules/macos_clang.lua @@ -1,14 +1,14 @@ help([[ This module loads libraries for building the CCPP Single-Column Model on -a MacOS machine with clang v15.0.0 compilers +a MacOS machine with clang compilers ]]) -whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang v15.0.0 compilers]===]) +whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang compilers]===]) local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") -load("stack-apple-clang/15.0.0") +load("stack-apple-clang") load("stack-python/3.10.13") load("stack-openmpi/4.1.6") From a93a5a527103e2c74239be6372f68f5f0c5bcc76 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 11:02:16 -0600 Subject: [PATCH 180/189] More quick-start updates - Move MacOS example out to an external link (Github Discussion) - Move downloading of static data after build - Convert "NOTE" asides into proper Sphinx Note blocks - More details about debug mode - Remove yet another forum mention, how are these still out there!? --- scm/doc/TechGuide/chap_quick.rst | 169 +++++++++++++++---------------- 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index c54ff8f8c..e5be962a2 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -205,47 +205,14 @@ Installing Libraries on Non-preconfigured Platforms For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. +The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole +Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may +install libraries manually if they wish, but they will need to make sure the appropriate environment variables +are set to the correct values so that the build system can find them: - -MacOS example -"""""""""""""" - -As an example of the steps needed for installing spack-stack on a MacOS machine, here is the list of steps that -resulted in a successful build and run of the SCM on a MacBook Pro with the Ventura 13.6.3 operating system, -using ``clang`` compilers. The necessary prerequisites for spack-stack (see -`spack-stack documentation `__) -were installed using Homebrew. - -:: - - cd your_experiment_dir - export TOP_DIR=`pwd` - mkdir spack_stack_1.6.0 - cd spack_stack_1.6.0 - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git -b 1.6.0 - cd spack-stack - source setup.sh - ulimit -S -s unlimited - spack stack create env --site macos.default --template ufs-weather-model --name SCM - cd envs/SCM - spack env activate -p . - export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" - spack external find --scope system --exclude bison --exclude openssl --exclude python - spack external find --scope system libiconv - spack external find --scope system perl - spack external find --scope system wget - PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" spack external find --scope system curl - PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" spack external find --scope system qt - spack config --scope system add packages:pkg-config:buildable:false - spack compiler find --scope system - unset SPACK_SYSTEM_CONFIG_PATH - spack config add "packages:all:compiler:[apple-clang@15.0.0]" - spack config add "packages:all:providers:mpi:[openmpi@4.1.6]" - spack concretize 2>&1 | tee log.concretize - ../../util/show_duplicate_packages.py -d log.concretize - spack install --verbose --fail-fast 2>&1 | tee log.install - spack module lmod refresh - spack stack setup-meta-modules +The CCPP/SCM team does not support spack-stack, so users with questions or requiring help with spack-stack installation +should reference the spack-stack documentation. However, we have provided an example procedure in +`this GitHub discussion `__. Setting up compilation environment @@ -254,12 +221,26 @@ Setting up compilation environment Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables needed for specifying the location of the various prerequisites. Users will need to set variables for the compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), -``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the +provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software +and set these variables. + +For users on a pre-configured platform, you can load the spack-stack environment via one of the provided modules in ``scm/etc/modules/``. +For example, users on the NSF NCAR machine Derecho who wish to use Intel compilers can do the following: + +:: + + cd [path/to/ccpp-scm/] + module use scm/etc/modules/ + module load derecho_intel + +Additionally, for users who have installed spack-stack on their own MacOS or Linux machine can use the provided ``macos_clang`` +or ``linux_gnu`` modules. + +.. note:: -The SCM uses only a small part of the UFS *hpc-stack* package and has fewer -prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use the -UFS can install only NetCDF/NetCDF-Fortran manually or using the -software package manager (apt, yum, brew). + The provided modules assume ``clang``/``gfortran`` compilers on MacOS and GNU compilers for Linux. + If you are using a different set of compilers, you may need to modify the module file. Python requirements """"""""""""""""""""" @@ -298,24 +279,7 @@ Compiling SCM with CCPP The first step in compiling the CCPP and SCM is to properly setup your user environment as described in -sections :numref:`%s ` and :numref:`Section %s `. The second step is -to download the lookup tables and other large datasets (large binaries, -:math:`<`\ 1 GB) needed by the physics schemes and place them in the -correct directory: From the top-level code directory (``ccpp-scm`` by default), -execute the following scripts: - -.. code:: bash - - ./contrib/get_all_static_data.sh - ./contrib/get_thompson_tables.sh - -If the download step fails, make sure that your system’s firewall does -not block access to GitHub. If it does, download the files ``comparison_data.tar.gz``, -``physics_input_data.tar.gz``, ``processed_case_input.tar.gz``, and ``raw_case_input.tar.gz`` -from the GitHub release website using your browser and manually extract its -contents in the directory ``scm/data``. Similarly, do the same for -``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract -to ``scm/data/physics_input_data/``. +sections :numref:`%s ` and :numref:`Section %s `. Following this step, the top level build system will use ``cmake`` to query system parameters, execute the CCPP prebuild script to match the physics @@ -379,7 +343,8 @@ components. -DOPENMP=ON - - Debug mode + - Debug mode, which compiles with lower optimization and additional compile-time checks. Only + recommended for development and debugging, because code compiled in this mode will run slower. .. code:: bash @@ -433,12 +398,13 @@ directory) pwd #confirm that you are in the ccpp-scm/scm/bin directory before deleting files rm -rfd * -Note: This command can be dangerous (deletes files without confirming), -so make sure that you’re in the right directory before executing! +.. note:: + This command can be dangerous (deletes files without confirming), + so make sure that you’re in the right directory before executing! If you encounter errors, please capture a log file from all of the -steps, and start a thread on the support forum at: -https://dtcenter.org/forum/ccpp-user-support/ccpp-single-column-model +steps, and start a thread on the Github Discussions support forum at: +https://github.com/NCAR/ccpp-scm/discussions Run the SCM with a supplied case -------------------------------- @@ -451,6 +417,27 @@ executed through a Python run script that is pre-staged into the ``bin`` directory: ``run_scm.py``. It can be used to run one integration or several integrations serially, depending on the command line arguments supplied. +Downloading input data +^^^^^^^^^^^^^^^^^^^^^^ +The various SCM cases require staged input data in order to run. This includes +input data for cases and lookup tables for runtime use. This is a large dataset +(:math:`<`\ 1 GB) so it is not stored in the SCM repository, and must be downloaded +separately. To download this data place it in the correct directories, +execute the following scripts: + +.. code:: bash + + ./contrib/get_all_static_data.sh + ./contrib/get_thompson_tables.sh + +If the download step fails, make sure that your system’s firewall does +not block access to GitHub. If it does, download the files ``comparison_data.tar.gz``, +``physics_input_data.tar.gz``, ``processed_case_input.tar.gz``, and ``raw_case_input.tar.gz`` +from the `SCM release page `__ using your browser and manually extract its +contents in the directory ``scm/data``. Similarly, do the same for +``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract +to ``scm/data/physics_input_data/``. + .. _`singlerunscript`: Run Script Usage @@ -709,10 +696,12 @@ In order to run a precompiled version of the CCPP SCM in a container, Docker will need to be available on your machine. Please visit https://www.docker.com to download and install the version compatible with your system. Docker frequently releases updates to the software; it -is recommended to apply all available updates. NOTE: In order to install -Docker on your machine, you will be required to have root access -privileges. More information about getting started can be found at -https://docs.docker.com/get-started +is recommended to apply all available updates. + +.. note:: + In order to install Docker on your machine, you will be required to have root access + privileges. More information about getting started can be found at + https://docs.docker.com/get-started The following tips were acquired during a recent installation of Docker on a machine with Windows 10 Home Edition. Further help should be @@ -775,8 +764,9 @@ can be generated from the code in the software repository obtained by following the instructions in :numref:`Section %s `, and then executing the following steps: -NOTE: Windows users can execute these steps in the terminal application -that was installed as part of Docker Toolbox. +.. note:: + Windows users can execute these steps in the terminal application + that was installed as part of Docker Toolbox. #. Navigate to the ``ccpp-scm/docker`` directory. @@ -818,8 +808,9 @@ To verify that it exists afterward, run Running the Docker image ^^^^^^^^^^^^^^^^^^^^^^^^ -NOTE: Windows users can execute these steps through the Docker -Quickstart application installed with Docker Toolbox. +.. note:: + Windows users can execute these steps through the Docker + Quickstart application installed with Docker Toolbox. #. Set up a directory that will be shared between the host machine and the Docker container. When set up correctly, it will contain output @@ -870,9 +861,13 @@ Quickstart application installed with Docker Toolbox. docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -c twpice -d will run through the TWPICE case using the default suite and namelist - and put the output in the shared directory. NOTE: Windows users may - need to omit the curly braces around environment variables: use ``$OUT_DIR`` - instead of ``${OUT_DIR}``. For running through all supported cases and suites, use + and put the output in the shared directory. + + .. note:: + Windows users may need to omit the curly braces around environment variables: use ``$OUT_DIR`` + instead of ``${OUT_DIR}``. + + For running through all supported cases and suites, use .. code:: bash @@ -894,9 +889,10 @@ Quickstart application installed with Docker Toolbox. - ``−−name`` names the container. If no name is provided, the daemon will autogenerate a random string name. - NOTE: If you are using a prebuilt image from Dockerhub, substitute - the name of the image that was pulled from Dockerhub in the commands - above; i.e. instead of ``ccpp-scm`` above, one would have ``dtcenter/ccpp-scm:v6.0.0``. + .. note:: + If you are using a prebuilt image from Dockerhub, substitute + the name of the image that was pulled from Dockerhub in the commands + above; i.e. instead of ``ccpp-scm`` above, one would have ``dtcenter/ccpp-scm:v6.0.0``. #. To use the SCM interactively, run non-default configurations, create plots, or even develop code, issue the following command: @@ -909,6 +905,9 @@ Quickstart application installed with Docker Toolbox. directory of the SCM with a pre-compiled executable. At this point, one could use the run scripts as described in previous sections (remembering to include the option on run scripts if output is to be - shared with the host machine). NOTE: If developing, since the - container is ephemeral, one should push their changes to a remote git - repository to save them (i.e. a fork on GitHub.com). + shared with the host machine). + + .. note:: + + If developing or modifying code, since the container is ephemeral, one should push their changes to a remote git + repository to save them (i.e. a fork on GitHub.com). From 18fb473e22f2de3513ac61a2c08695e50ce62d1d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 15:51:21 -0600 Subject: [PATCH 181/189] Minor users guide updates --- scm/doc/TechGuide/chap_quick.rst | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index e5be962a2..6b77a9447 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -156,7 +156,7 @@ contains the following set of libraries needed for building the SCM: Instructions for installing spack-stack can be found in the `spack-stack documentation `__. Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and -Jet, and MSU's Orion and Hercules. +Jet, and MSU's Orion. Compilers ^^^^^^^^^ @@ -205,26 +205,21 @@ Installing Libraries on Non-preconfigured Platforms For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. -The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole -Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may -install libraries manually if they wish, but they will need to make sure the appropriate environment variables -are set to the correct values so that the build system can find them: The CCPP/SCM team does not support spack-stack, so users with questions or requiring help with spack-stack installation -should reference the spack-stack documentation. However, we have provided an example procedure in +should reference the `spack-stack documentation `__. +However, we have provided an example procedure in `this GitHub discussion `__. +The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole +Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may +install libraries manually if they wish, but they will need to make sure the appropriate environment variables +are set to the correct values so that the build system can find them, as described in the following chapter. + Setting up compilation environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables -needed for specifying the location of the various prerequisites. Users will need to set variables for the -compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), -``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the -provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software -and set these variables. - For users on a pre-configured platform, you can load the spack-stack environment via one of the provided modules in ``scm/etc/modules/``. For example, users on the NSF NCAR machine Derecho who wish to use Intel compilers can do the following: @@ -242,6 +237,14 @@ or ``linux_gnu`` modules. The provided modules assume ``clang``/``gfortran`` compilers on MacOS and GNU compilers for Linux. If you are using a different set of compilers, you may need to modify the module file. +If libraries were installed manually, users will need to set some environment variables +needed for specifying the location of the various prerequisites. Users will need to set variables for the +compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the +provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software +and set these variables. + + Python requirements """"""""""""""""""""" From daf85633e006387b3f4f8b8860f289060bdfb09e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 1 May 2024 15:52:32 -0600 Subject: [PATCH 182/189] Consolidate some repeated directory setup commands in Dockerfile --- docker/Dockerfile | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9699254c4..6212a051a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,17 +23,13 @@ ENV F90 /usr/bin/gfortran # Other necessary environment variables ENV LD_LIBRARY_PATH /usr/lib/ -# Set up unpriviledged user account -RUN groupadd comusers -g 9999 -RUN useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser -RUN mkdir /comsoftware \ - && chown -R comuser:comusers /comsoftware \ - && chmod 6755 /comsoftware - -# Set up user home space correctly and make sure user has permissions on all stuff in /comsoftware -RUN chown -R comuser:comusers /home \ +# Set up unpriviledged user account, set up user home space and make sure user has permissions on all stuff in /comsoftware +RUN groupadd comusers -g 9999 \ + && useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser \ + && mkdir /comsoftware \ + && chown -R comuser:comusers /home \ && chmod 6755 /home -RUN chown -R comuser:comusers /comsoftware \ + && chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware # Link version-specific aliases (python3 will be created later with virtual environment) From 755e0c51f8e8ffcb6a264056cc5b9cf617b54848 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 1 May 2024 15:54:13 -0600 Subject: [PATCH 183/189] More Quick Start updates - Update SCM run commands, including some examples - Remove the deprecated "multirun" argument - Update argument descriptions - Update description of Docker software stack --- scm/doc/TechGuide/chap_quick.rst | 93 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 6b77a9447..958868cf3 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -455,16 +455,33 @@ namelists in ``../etc/case_config``. A default physics suite is provided as a us variable in the script and default namelists and tracer configurations are associated with each physics suite (through ``../src/suite_info.py``), so, technically, one must only specify a case to run with the SCM when running just one -integration. For running multiple integrations at once, one need only -specify one argument (``-m``) which runs through all permutations of supported -suites from ``../src/suite_info.py`` and cases from ``../src/supported_cases.py``. The run script’s options are described -below where option abbreviations are included in brackets. +integration. For example, to run the "BOMEX" case: + +.. code:: bash + + ./run_scm.py -c bomex + +For running multiple integrations at once, the run script can accept a file that contains a list of tests to run. +The file ``ccpp-scm/test/rt_test_cases.py`` contains the full list of regression test cases, so you could run that list +of tests with the following command: + + ./run_scm.py -f ../../test/rt_test_cases.py + +To see the full list of available options, use the ``--help`` flag: + +../../test/rt_test_cases.py +.. code:: bash + + ./run_scm.py --help + + +The run script’s full set of options are described below, where optional abbreviations are included in brackets. +If using the main branch, you should run the above command to ensure you have the most up-to-date list of options. - ``--case [-c]`` - - **This or the ``--multirun`` option are the minimum required arguments.** The - case should correspond to the name of a case in ``../etc/case_config`` (without the - ``.nml`` extension). + - **This is the only required argument.** The provided argument should correspond to the name of a case in + ``../etc/case_config`` (without the ``.nml`` extension). - ``--suite [-s]`` @@ -483,18 +500,9 @@ below where option abbreviations are included in brackets. the ``.txt`` extension). If this argument is omitted, the default tracer configuration for the given suite in ``../src/suite_info.py`` will be used. -- ``--multirun [-m]`` - - - **This or the ``--case`` option are the minimum required arguments.** When - used alone, this option runs through all permutations of supported - suites from ``../src/suite_info.py`` and cases from ``../src/supported_cases.py``. When used in conjunction with the - ``--file`` option, only the runs configured in the file will be run. - - ``--file [-f]`` - - This option may be used in conjunction with the ``--multirun`` argument. It - specifies a path and filename to a python file where multiple runs - are configured. + - This option may be used to specify a list of tests to run; see ../../test/rt_test_cases.py for an example. - ``--gdb [-g]`` @@ -505,7 +513,7 @@ below where option abbreviations are included in brackets. - Use this argument when running in a docker container in order to successfully mount a volume between the host machine and the - Docker container instance and to share the output and plots with + Docker container instance, allowing the container to share the output and plots with the host machine. - ``--runtime`` @@ -518,7 +526,7 @@ below where option abbreviations are included in brackets. - Use this to override the runtime provided in the case configuration namelist by multiplying the runtime by the given value. This is used, for example, in regression testing to reduce - total runtimes. + total runtimes (e.g., ``--runtime_mult 0.1``). - ``--levels [-l]`` @@ -574,7 +582,7 @@ configuration files located in ``../etc/case_config`` (*without the .nml extensi specifying a suite other than the default, the suite name used must match the value of the suite name in one of the suite definition files located in ``../../ccpp/suites`` (Note: not the filename of the suite definition file). As -part of the sixth CCPP release, the following suite names are valid: +part of the sixth CCPP release, the following suite names are supported: #. SCM_GFS_v16 @@ -735,32 +743,28 @@ Building the Docker image ^^^^^^^^^^^^^^^^^^^^^^^^^ The Dockerfile builds CCPP SCM v6.0.0 from source using the GNU -compiler. A number of required codes are built and installed via the -DTC-supported common community container. For reference, the common -community container repository can be accessed here: -https://github.com/NCAR/Common-Community-Container. +compiler. The CCPP SCM has a number of system requirements and necessary libraries and tools. Below is a list, including versions, used to create the the -GNU-based Docker image: +GNU-based Docker image. These are included for reference, but recall that +the Docker container contains all of this software built-in, you do not need to install them separately! -- gfortran - 9.3 +- gfortran - 12.2.0 -- gcc - 9.3 +- gcc - 12.2.0 -- cmake - 3.16.5 +- cmake - 3.25.1 -- NetCDF - 4.6.2 +- NetCDF - 4.9.0 -- HDF5 - 1.10.4 +- Python - 3.11.2 -- ZLIB - 1.2.7 +- NCEPLIBS BACIO - v2.4.1 -- SZIP - 2.1.1 +- NCEPLIBS SP - v2.3.3 -- Python - 3 - -- NCEPLIBS subset: bacio v2.4.1_4, sp v2.3.3_d, w3emc v2.9.2_d +- NCEPLIBS W3EMC - v2.11.0 A Docker image containing the SCM, CCPP, and its software prerequisites can be generated from the code in the software repository obtained by @@ -783,13 +787,15 @@ and then executing the following steps: Inspect the Dockerfile if you would like to see details for how the image is built. The image will contain SCM prerequisite software from DTC, the SCM and CCPP code, and a pre-compiled executable for the SCM - with the 6 supported suites for the SCM. A successful build will show - two images: dtcenter/common-community-container, and ccpp-scm. To - list images, type: + with the 6 supported suites for the SCM. To view .. code:: bash - docker images + > docker images + + REPOSITORY TAG IMAGE ID CREATED SIZE + ccpp-scm latest 1b2e0a0afdf9 2 days ago 3.21GB + Using a prebuilt Docker image from Dockerhub ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -838,14 +844,7 @@ Running the Docker image does not) and make sure that the “auto-mount" and “permanent" options are checked. -#. Set an environment variable to use for your SCM output directory. For - *t/csh* shells, - - .. code:: bash - - setenv OUT_DIR /path/to/output - - For bourne/bash shells, +#. Set an environment variable to use for your SCM output directory. .. code:: bash From 6cb6db864990cf3bcdd62c423fb256f1fe84bb34 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:00:02 -0600 Subject: [PATCH 184/189] Fix format error, convert some important notes to warnings --- scm/doc/TechGuide/chap_quick.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 958868cf3..cbaf53acc 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -401,7 +401,7 @@ directory) pwd #confirm that you are in the ccpp-scm/scm/bin directory before deleting files rm -rfd * -.. note:: +.. warning:: This command can be dangerous (deletes files without confirming), so make sure that you’re in the right directory before executing! @@ -465,11 +465,12 @@ For running multiple integrations at once, the run script can accept a file that The file ``ccpp-scm/test/rt_test_cases.py`` contains the full list of regression test cases, so you could run that list of tests with the following command: +.. code:: bash + ./run_scm.py -f ../../test/rt_test_cases.py To see the full list of available options, use the ``--help`` flag: -../../test/rt_test_cases.py .. code:: bash ./run_scm.py --help @@ -909,7 +910,7 @@ Running the Docker image (remembering to include the option on run scripts if output is to be shared with the host machine). - .. note:: + .. warning:: If developing or modifying code, since the container is ephemeral, one should push their changes to a remote git repository to save them (i.e. a fork on GitHub.com). From 1d2f9e705c69d87370c58299b9060a374ed24e39 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:20:06 -0600 Subject: [PATCH 185/189] Fix incorrect Dockerhub org name --- .github/workflows/build_and_push_docker_latest.yml | 2 +- .github/workflows/ci_test_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml index d9dd73192..487933e39 100644 --- a/.github/workflows/build_and_push_docker_latest.yml +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -7,7 +7,7 @@ on: - "main" env: - LATEST_TAG: NCAR/ccpp-scm:latest + LATEST_TAG: dtcenter/ccpp-scm:latest jobs: docker: diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index 1fac0d699..a5d36a6f3 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -3,7 +3,7 @@ name: build_test_and_push_docker on: [pull_request,workflow_dispatch] env: - TEST_TAG: NCAR/ccpp-scm:test + TEST_TAG: dtcenter/ccpp-scm:test jobs: docker: From 074fbd867be70ce28f1fc49554c96ae4fb25374b Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:30:31 -0600 Subject: [PATCH 186/189] Apparently GitHub secrets are not passed when PRs come in from forks. This job shouldn't need those credentials anyway now that I separated out the push step, so removing the Login step should fix this --- .github/workflows/ci_test_docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index a5d36a6f3..3406d8326 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -15,11 +15,6 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and export test tag uses: docker/build-push-action@v5 with: From 225706201df3a627e446d467311967739ee8d3c6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:35:40 -0600 Subject: [PATCH 187/189] Fix dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6212a051a..7d0586090 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,7 +28,7 @@ RUN groupadd comusers -g 9999 \ && useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser \ && mkdir /comsoftware \ && chown -R comuser:comusers /home \ - && chmod 6755 /home + && chmod 6755 /home \ && chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware From 0283c188d8dcd4d21a794c5da8f007884e57abfe Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 15 May 2024 11:46:05 -0600 Subject: [PATCH 188/189] Updates from Dustin's comments --- scm/doc/TechGuide/chap_quick.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index cbaf53acc..9f407dbba 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -76,7 +76,7 @@ developers. While the SCM is updated with the latest commits to the CCPP submodules (ccpp-physics and ccpp-framework) -on a fairly regular basis, it may be weeks or even months behind at times. You can try to use the latest commits of the ccpp-physics and +on a fairly regular basis, it may be behind by a few commits at times. You can try to use the latest commits of the ccpp-physics and ccpp-framework submodules if you wish, but this may not have been tested (i.e. SCM development may lag ccpp-physics and/or ccpp-framework development). To do so: @@ -441,6 +441,12 @@ contents in the directory ``scm/data``. Similarly, do the same for ``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract to ``scm/data/physics_input_data/``. +New with the SCM v7 release, static data is available for running cases with GOCART climatological aerosols (where the value of ``iaer`` in the ``&gfs_physics_nml`` namelist starts with 1; see the `CCPP Scientific Documentation `__ for more information); one example of this is with the default namelist settings for the GFS_v17_HR3 scheme. This dataset is very large (~12 GB), so it is recommended only to download it if you will be using it. + +.. code:: bash + + ./contrib/get_aerosol_climo.sh + .. _`singlerunscript`: Run Script Usage From 10c8f0578cab49d1580b3dd5e3012255c953e92b Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 22 May 2024 10:38:29 -0600 Subject: [PATCH 189/189] Run build and push docker test on ubuntu-22.04 per concerns about "latest" tag --- .github/workflows/build_and_push_docker_latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml index 487933e39..4f04b7ec5 100644 --- a/.github/workflows/build_and_push_docker_latest.yml +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -11,7 +11,7 @@ env: jobs: docker: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4