"update to address code reviewers' comments"

physics/cu_gf_deep.F90

@@ -223,6 +223,7 @@ subroutine cu_gf_deep_run(        &
      real(kind=kind_phys), dimension (its:ite,nchem)                   &
          , intent (out) :: wetdpc_deep
      real(kind=kind_phys), intent (in) :: fscav(:)
+!$acc declare copy(chem3d) copyout(wetdpc_deep) copyin(fscav)
 
      real(kind=kind_phys)                                              &
         ,intent (in   )                   ::                           &
@@ -314,6 +315,8 @@ subroutine cu_gf_deep_run(        &
      real(kind=kind_phys), dimension (kts:kte) :: trac,trcflx_in,trcflx_out,trc,trco
      real(kind=kind_phys), dimension (its:ite,kts:kte) :: pwdper, massflx
      integer :: nv
+!$acc declare create(chem,chem_cup,chem_up,chem_down,dellac,dellac2,chem_c,chem_pw,chem_pwd,   &
+!$acc                         chem_pwav,chem_psum,pwdper,massflux)
 
      real(kind=kind_phys),    dimension (its:ite,kts:kte) ::            &
         entr_rate_2d,mentrd_rate_2d,he,hes,qes,z, heo,heso,qeso,zo,     &                    
@@ -424,7 +427,7 @@ subroutine cu_gf_deep_run(        &
      integer, dimension (its:ite,kts:kte) ::  k_inv_layers 
      real(kind=kind_phys),    dimension (its:ite) :: c0    ! HCB
      real(kind=kind_phys),    dimension (its:ite,kts:kte) :: c0t3d    ! hli for smoke/dust wet scavenging
-!$acc declare create(pmin_lev,start_level,ktopkeep,dtempdz,k_inv_layers,c0)
+!$acc declare create(pmin_lev,start_level,ktopkeep,dtempdz,k_inv_layers,c0,c0t3d)
 
 ! rainevap from sas
      real(kind=kind_phys) zuh2(40)
@@ -2046,6 +2049,7 @@ subroutine cu_gf_deep_run(        &
 ! initialize tracers if they exist
 !
          chem (:,:,:) = 0.
+!$acc kernels
          do nv = 1,nchem
           do k = 1, ktf
            do i = 1, itf
@@ -2069,7 +2073,7 @@ subroutine cu_gf_deep_run(        &
          do i=its,itf
            if(ierr(i).eq.0)then
            do k=kts,jmin(i)
-              pwdper(i,k)=-edtc(i,1)*pwdo(i,k)/pwavo(i)
+             if(pwavo(i).ne.0.) pwdper(i,k)=-edtc(i,1)*pwdo(i,k)/pwavo(i)
            enddo
            pwdper(i,:)=0.
            do nv=1,nchem
@@ -2094,8 +2098,6 @@ subroutine cu_gf_deep_run(        &
                 trash=chem_c(i,k,nv)/(1.+c0t3d(i,k)*dz)
                 chem_pw=c0t3d(i,k)*dz*trash*zuo(i,k)
                 chem_up(i,k,nv)=trash2+trash
-!               chem_pw(i,k,nv)=min(chem_up(i,k,nv),chem_c(i,k,nv)*pwo(i,k)/zuo(i,k)/(1.e-8+qrco(i,k)))
-!               chem_up(i,k,nv)=chem_up(i,k,nv)-chem_pw(i,k,nv)
                 chem_pwav(i,nv)=chem_pwav(i,nv)+chem_pw(i,k,nv)! *g/dp
               enddo
               do k=ktop(i)+1,ktf
@@ -2109,11 +2111,11 @@ subroutine cu_gf_deep_run(        &
                do ki=jmin(i),2,-1
                  dp=100.*(po_cup(i,ki)-po_cup(i,ki+1))
                  chem_down(i,ki,nv)=(chem_down(i,ki+1,nv)*zdo(i,ki+1)          &
-                       -.5*dd_massdetro(i,ki)*chem_down(i,ki+1,nv)+            &
+                       -.5_kind_phys*dd_massdetro(i,ki)*chem_down(i,ki+1,nv)+            &
                        dd_massentro(i,ki)*chem(i,ki,nv))   /                   &
-                       (zdo(i,ki+1)-.5*dd_massdetro(i,ki)+dd_massentro(i,ki))
+                       (zdo(i,ki+1)-.5_kind_phys*dd_massdetro(i,ki)+dd_massentro(i,ki))
                  chem_down(i,ki,nv)=chem_down(i,ki,nv)+pwdper(i,ki)*chem_pwav(i,nv)
-                 chem_pwd(i,ki,nv)=max(0.,pwdper(i,ki)*chem_pwav(i,nv))
+                 chem_pwd(i,ki,nv)=max(0._kind_phys,pwdper(i,ki)*chem_pwav(i,nv))
                enddo
 !   total wet deposition
                do k=1,ktf-1
@@ -2167,15 +2169,16 @@ subroutine cu_gf_deep_run(        &
       dellac2(:,:,:)=0.
       massflx(:,:)=0.
       do nv=1,nchem
+!$acc loop private(trcflx_in)
       do i=its,itf
         if(ierr(i).eq.0)then
          trcflx_in(:)=0.
          dtime_max=dtime
 
 ! initialize fct routine
          do k=kts,ktop(i)
-            dp=100.*(po_cup(i,k)-po_cup(i,k+1))
-            dtime_max=min(dtime_max,.5*dp)
+            dp=100._kind_phys*(po_cup(i,k)-po_cup(i,k+1))
+            dtime_max=min(dtime_max,.5_kind_phys*dp)
             massflx(i,k)=-xmb(i)*(zuo(i,k)-edto(i)*zdo(i,k))
             trcflx_in(k)=massflx(i,k)*chem_cup(i,k,nv)
          enddo
@@ -2212,6 +2215,7 @@ subroutine cu_gf_deep_run(        &
           wetdpc_deep(i,nv)=max(wetdpc_deep(i,nv),qamin)
          enddo
         enddo
+!$acc end kernels
 
       endif ! nchem > 0
 
@@ -4345,6 +4349,7 @@ subroutine cup_up_moisture(name,ierr,z_cup,qc,qrc,pw,pwav,     &
         ,intent (in   )                   ::                      &
         kbcon,ktop,k22,xland1
 !$acc declare copyin(p_cup,rho,q,zu,gamma_cup,qe_cup,up_massentr,up_massdetr,dby,qes_cup,z_cup,zqexec,c0,kbcon,ktop,k22,xland1)
+!$acc declare copy(c0t3d)
      real(kind=kind_phys),    intent (in  ) ::                    & ! HCB
         ccnclean
 !
@@ -4421,7 +4426,9 @@ subroutine cup_up_moisture(name,ierr,z_cup,qc,qrc,pw,pwav,     &
         c0_iceconv=0.01
         c1d_b=c1d
         bdsp(:)=bdispm
+!$acc kernels
         c0t3d = 0.
+!$acc end kernels
 
 !
 !--- no precip for small clouds

physics/cu_gf_driver.F90

@@ -155,9 +155,10 @@ subroutine cu_gf_driver_run(ntracer,garea,im,km,dt,flag_init,flag_restart,&
    integer, intent(in   ) :: imfshalcnv
    integer, dimension(:), intent(inout) :: cactiv,cactiv_m
    real(kind_phys), dimension(:), intent(in) :: fscav
+!$acc declare copyin(fscav)
    real(kind_phys), dimension(:,:,:), intent(inout) :: chem3d
    real(kind_phys), dimension(:,:), intent(inout) :: wetdpc_deep
-!$acc declare copy(cactiv,cactiv_m)
+!$acc declare copy(cactiv,cactiv_m,chem3d,wetdpc_deep)
 
    character(len=*), intent(out) :: errmsg
    integer,          intent(out) :: errflg
@@ -188,14 +189,14 @@ subroutine cu_gf_driver_run(ntracer,garea,im,km,dt,flag_init,flag_restart,&
    integer, dimension (im) :: k22s,kbcons,ktops,k22,jmin,jminm
    integer, dimension (im) :: kbconm,ktopm,k22m
 !$acc declare create(k22_shallow,kbcon_shallow,ktop_shallow,rand_mom,rand_vmas,        &
-!$acc                rand_clos,gdc,gdc2,ht,ccn_gf,ccn_m,dx,frhm,frhd, &
+!$acc                rand_clos,gdc,gdc2,ht,ccn_gf,ccn_m,dx,frhm,frhd,wetdpc_mid, &
 !$acc                outt,outq,outqc,phh,subm,cupclw,cupclws, &
 !$acc                dhdt,zu,zus,zd,phf,zum,zdm,outum,outvm,   &
 !$acc                outts,outqs,outqcs,outu,outv,outus,outvs, &
 !$acc                outtm,outqm,outqcm,submm,cupclwm,         &
 !$acc                cnvwt,cnvwts,cnvwtm,hco,hcdo,zdo,zdd,hcom,hcdom,zdom, &
 !$acc                tau_ecmwf,edt,edtm,edtd,ter11,aa0,xlandi, &
-!$acc                pret,prets,pretm,hexec,forcing,forcing2,  &
+!$acc                pret,prets,pretm,hexec,forcing,forcing2,wetdpc_mid,  &
 !$acc                kbcon, ktop,ierr,ierrs,ierrm,kpbli, &
 !$acc                k22s,kbcons,ktops,k22,jmin,jminm,kbconm,ktopm,k22m)
 

physics/cu_gf_driver.meta

@@ -614,7 +614,7 @@
   intent = in
 [nchem]
   standard_name = number_of_chemical_species_vertically_mixed
-  long_name = number of chemical vertically mixed
+  long_name = number of chemical species vertically mixed
   units = count
   dimensions = ()
   type = integer

physics/cu_gf_driver_post.F90

@@ -35,7 +35,7 @@ subroutine cu_gf_driver_post_run (im, km, t, q, prevst, prevsq, cactiv, cactiv_m
       integer,          intent(in)  :: ntsmoke, ntdust, ntcoarsepm
       real(kind_phys),  intent(inout) :: chem3d(:,:,:), gq0(:,:,:)
       character(len=*), intent(out) :: errmsg
-!$acc declare copyin(t,q,cactiv,cactiv_m) copyout(prevst,prevsq,conv_act,conv_act_m)
+!$acc declare copyin(t,q,cactiv,cactiv_m) copyout(prevst,prevsq,conv_act,conv_act_m,chem3d,gq0)
       integer, intent(out)          :: errflg
 
       ! Local variables
@@ -61,13 +61,13 @@ subroutine cu_gf_driver_post_run (im, km, t, q, prevst, prevsq, cactiv, cactiv_m
           conv_act_m(i)=0.0
         endif
       enddo
-!$acc end kernels
 
       if (rrfs_sd) then
        gq0(:,:,ntsmoke   ) = chem3d(:,:,1)
        gq0(:,:,ntdust    ) = chem3d(:,:,2)
        gq0(:,:,ntcoarsepm) = chem3d(:,:,3)
       endif
+!$acc end kernels
 
    end subroutine cu_gf_driver_post_run
 

physics/cu_gf_driver_post.meta

@@ -85,7 +85,7 @@
   intent = out
 [rrfs_sd]
   standard_name = do_smoke_coupling
-  long_name = flag controlling rrfs_sd collection (default off)
+  long_name = flag controlling rrfs_sd collection
   units = flag
   dimensions = ()
   type = logical

physics/cu_gf_driver_pre.F90

@@ -44,7 +44,7 @@ subroutine cu_gf_driver_pre_run (flag_init, flag_restart, kdt, fhour, dtp, t, q,
       real(kind_phys),  intent(in)  :: conv_act(:)
       real(kind_phys),  intent(in)  :: conv_act_m(:)
       real(kind_phys),  intent(inout) :: chem3d(:,:,:), gq0(:,:,:)
-!$acc declare copyin(conv_act,conv_act_m)
+!$acc declare copyin(conv_act,conv_act_m) copy(chem3d,gq0)
       character(len=*), intent(out) :: errmsg
       integer,          intent(out) :: errflg
 
@@ -81,13 +81,13 @@ subroutine cu_gf_driver_pre_run (flag_init, flag_restart, kdt, fhour, dtp, t, q,
 !$acc kernels
       cactiv(:)=nint(conv_act(:))
       cactiv_m(:)=nint(conv_act_m(:))
-!$acc end kernels
 
       if (rrfs_sd) then
        chem3d(:,:,1) = gq0(:,:,ntsmoke)
        chem3d(:,:,2) = gq0(:,:,ntdust)
        chem3d(:,:,3) = gq0(:,:,ntcoarsepm)
       endif
+!$acc end kernels
 
    end subroutine cu_gf_driver_pre_run
 

physics/cu_gf_driver_pre.meta

@@ -124,7 +124,7 @@
   intent = in
 [rrfs_sd]
   standard_name = do_smoke_coupling
-  long_name = flag controlling rrfs_sd collection (default off)
+  long_name = flag controlling rrfs_sd collection
   units = flag
   dimensions = ()
   type = logical

physics/smoke_dust/coarsepm_settling_mod.F90

@@ -64,7 +64,6 @@ SUBROUTINE coarsepm_settling_driver(dt,t_phy,                         &
           airmas(1,1,kk)=-(p8w(i,k+1,j)-p8w(i,k,j))*area(i,j)/g
           airden(1,1,kk)=rho_phy(i,k,j)
           tmp(1,1,kk)=t_phy(i,k,j)
-!          rh(1,1,kk) = rel_hum(i,k,j) ! hli
           do nv = 1, num_chem
             chem_before(i,j,k,nv) =  chem(i,k,j,nv)  
           enddo