FIX

anandrdbz · Mar 8, 2024 · 1c51ebe · 1c51ebe
1 parent 253f666
commit 1c51ebe
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 42 deletions.
diff --git a/src/common/m_variables_conversion.fpp b/src/common/m_variables_conversion.fpp
@@ -81,7 +81,7 @@ module m_variables_conversion
     end interface ! ============================================================
 
     integer, public :: ixb, ixe, iyb, iye, izb, ize
-    !!$acc declare create(ixb, ixe, iyb, iye, izb, ize)
+    !$acc declare create(ixb, ixe, iyb, iye, izb, ize)
     real(kind(0d0)) :: temp
 
     !! In simulation, gammas, pi_infs, and qvs are already declared in m_global_variables
@@ -102,6 +102,7 @@ module m_variables_conversion
     !$acc declare create(bubrs, Gs, Res)
 #endif
     integer :: is1b, is2b, is3b, is1e, is2e, is3e
+    !$acc declare create(is1b, is2b, is3b, is1e, is2e, is3e)
 
     real(kind(0d0)), allocatable, dimension(:, :, :), public :: rho_sf !< Scalar density function
     real(kind(0d0)), allocatable, dimension(:, :, :), public :: gamma_sf !< Scalar sp. heat ratio function
@@ -625,7 +626,9 @@ contains
             end if
         end if
 #endif
-
+
+        !$acc enter data copyin(ixb, ixe, iyb, iye, izb, ize)
+        !$acc enter data copyin(is1b, is1e, is2b, is2e, is3b, is3e)
         !$acc update device(ixb, ixe, iyb, iye, izb, ize)
 
 #ifdef MFC_SIMULATION
@@ -1216,7 +1219,7 @@ contains
         is2b = is2%beg; is2e = is2%end
         is3b = is3%beg; is3e = is3%end
 
-        !$acc enter data copyin(is1b, is2b, is3b, is1e, is2e, is3e)
+        !$acc update device(is1b, is2b, is3b, is1e, is2e, is3e)
 
         ! Computing the flux variables from the primitive variables, without
         ! accounting for the contribution of either viscosity or capillarity

diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp
@@ -131,10 +131,6 @@ module m_global_parameters
     integer :: riemann_solver !< Riemann solver algorithm
     integer :: wave_speeds    !< Wave speeds estimation method
     integer :: avg_state      !< Average state evaluation method
-    logical :: relax          !< activate phase change
-    integer :: relax_model    !< Relaxation model
-    real(kind(0d0)) :: palpha_eps     !< trigger parameter for the p relaxation procedure, phase change model
-    real(kind(0d0)) :: ptgalpha_eps   !< trigger parameter for the pTg relaxation procedure, phase change model
     logical :: alt_soundspeed !< Alternate mixture sound speed
     logical :: null_weights   !< Null undesired WENO weights
     logical :: mixture_err    !< Mixture properties correction
@@ -147,8 +143,16 @@ module m_global_parameters
         !$acc declare create(num_dims, weno_polyn, weno_order)
     #:endif
 
-    !$acc declare create(mpp_lim, num_fluids, model_eqns, mixture_err, alt_soundspeed, avg_state, mapped_weno, mp_weno, weno_eps, hypoelasticity, relax, palpha_eps,ptgalpha_eps)
+    !$acc declare create(mpp_lim, num_fluids, model_eqns, mixture_err, alt_soundspeed, avg_state, mapped_weno, mp_weno, weno_eps, hypoelasticity)
+
+    logical :: relax          !< activate phase change
+    integer :: relax_model    !< Relaxation model
+    real(kind(0d0)) :: palpha_eps     !< trigger parameter for the p relaxation procedure, phase change model
+    real(kind(0d0)) :: ptgalpha_eps   !< trigger parameter for the pTg relaxation procedure, phase change model
 
+#ifndef _CRAYFTN
+    !$acc declare create(relax, relax_model, palpha_eps,ptgalpha_eps)
+#endif
 
 
     !> @name Boundary conditions (BC) in the x-, y- and z-directions, respectively
@@ -963,6 +967,14 @@ contains
         intxe = internalEnergies_idx%end
 
         !$acc update device(momxb, momxe, advxb, advxe, contxb, contxe, bubxb, bubxe, intxb, intxe, sys_size, buff_size, E_idx, alf_idx, strxb, strxe)
+        !$acc update device(m, n, p)
+
+        !$acc update device(alt_soundspeed, monopole, num_mono)
+        !$acc update device(dt, sys_size, buff_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, nb, weight, grid_geometry, cyl_coord, mapped_weno, mp_weno, weno_eps)
+
+        !$acc enter data copyin(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma)
+        !$acc enter data copyin(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam)
+        !$acc enter data copyin(dir_idx, dir_flg, dir_idx_tau)
 
         ! Allocating grid variables for the x-, y- and z-directions
         @:ALLOCATE_GLOBAL(x_cb(-1 - buff_size:m + buff_size))

diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp
@@ -148,8 +148,11 @@ contains
             polytropic, thermal, &
             integral, integral_wrt, num_integrals, &
             polydisperse, poly_sigma, qbmm, &
-            R0_type, file_per_process, relax, relax_model, &
-            palpha_eps, ptgalpha_eps
+#ifndef _CRAYFTN
+             relax, relax_model, &
+            palpha_eps, ptgalpha_eps, &
+#endif
+            R0_type, file_per_process
 
         ! Checking that an input file has been provided by the user. If it
         ! has, then the input file is read in, otherwise, simulation exits.
@@ -1097,9 +1100,9 @@ contains
         elseif (time_stepper == 3) then
             call s_3rd_order_tvd_rk(t_step, time_avg)
         end if
-
+#ifndef _CRAYFTN
         if (relax) call s_relaxation_solver(q_cons_ts(1)%vf)
-
+#endif
         ! Time-stepping loop controls
         if ((mytime + dt) >= finaltime) dt = finaltime - mytime
         t_step = t_step + 1
@@ -1251,7 +1254,9 @@ contains
 #endif
 
         if (hypoelasticity) call s_initialize_hypoelastic_module()
+#ifndef _CRAYFTN       
         if (relax) call s_initialize_phasechange_module()
+#endif
         call s_initialize_data_output_module()
         call s_initialize_derived_variables_module()
         call s_initialize_time_steppers_module()
@@ -1366,31 +1371,25 @@ contains
     subroutine s_initialize_gpu_vars()
         integer :: i
         !Update GPU DATA
-        !$acc update device(dt, dx, dy, dz, x_cc, y_cc, z_cc, x_cb, y_cb, z_cb)
-        !$acc update device(sys_size, buff_size)
-        !$acc update device(m, n, p)
-        !$acc update device(momxb, momxe, bubxb, bubxe, advxb, advxe, contxb, contxe, strxb, strxe)
         do i = 1, sys_size
             !$acc update device(q_cons_ts(1)%vf(i)%sf)
         end do
         if (qbmm .and. .not. polytropic) then
             !$acc update device(pb_ts(1)%sf, mv_ts(1)%sf)
         end if
-        !$acc enter data copyin(alt_soundspeed)
-        !$acc update device(dt, sys_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, nb, weight, grid_geometry, cyl_coord, mapped_weno, mp_weno, weno_eps)
         !$acc update device(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma)
         !$acc update device(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam)
-        !$acc update device(monopole, num_mono)
-
+        !$acc update device(dx, dy, dz, x_cb, x_cc, y_cb, y_cc, z_cb, z_cc)    
         !$acc update device(bc_x%vb1, bc_x%vb2, bc_x%vb3, bc_x%ve1, bc_x%ve2, bc_x%ve3)
         !$acc update device(bc_y%vb1, bc_y%vb2, bc_y%vb3, bc_y%ve1, bc_y%ve2, bc_y%ve3)
         !$acc update device(bc_z%vb1, bc_z%vb2, bc_z%vb3, bc_z%ve1, bc_z%ve2, bc_z%ve3)
 
-
+#ifndef _CRAYFTN
         !$acc update device(relax)
         if (relax) then
             !$acc update device(palpha_eps, ptgalpha_eps)
         end if
+#endif
     end subroutine s_initialize_gpu_vars
 
     subroutine s_finalize_modules()
@@ -1409,8 +1408,9 @@ contains
         if (grid_geometry == 3) call s_finalize_fftw_module
         call s_finalize_mpi_proxy_module()
         call s_finalize_global_parameters_module()
+#ifndef _CRAYFTN
         if (relax) call s_finalize_relaxation_solver_module()      
-
+#endif
         if (any(Re_size > 0)) then
             call s_finalize_viscous_module()
         end if

diff --git a/src/simulation/m_time_steppers.fpp b/src/simulation/m_time_steppers.fpp
@@ -456,9 +456,16 @@ contains
 
         if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
 
-        if (model_eqns == 3 .and. (.not. relax)) then
+        if (model_eqns == 3) then
+#ifdef _CRAYFTN
             call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
-        end if
+#else
+            if(.not. relax) then
+               call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
+            end if 
+#endif
+       end if
+
 
         if (ib) then
             if (qbmm .and. .not. polytropic) then
@@ -525,9 +532,16 @@ contains
 
         if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
 
-        if (model_eqns == 3 .and. (.not. relax)) then
+        if (model_eqns == 3 ) then
+#ifdef _CRAYFTN
             call s_pressure_relaxation_procedure(q_cons_ts(1)%vf)
-        end if
+#else
+            if(.not. relax) then
+               call s_pressure_relaxation_procedure(q_cons_ts(1)%vf)
+           end if 
+#endif
+       end if
+
 
         if (ib) then
             if (qbmm .and. .not. polytropic) then
@@ -630,9 +644,17 @@ contains
 
         if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
 
-        if (model_eqns == 3 .and. (.not. relax)) then
+
+        if (model_eqns == 3 ) then
+#ifdef _CRAYFTN
             call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
-        end if
+#else
+            if(.not. relax) then
+               call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
+           end if 
+#endif
+       end if
+
 
         if (ib) then
             if (qbmm .and. .not. polytropic) then
@@ -701,9 +723,16 @@ contains
 
         if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
 
-        if (model_eqns == 3 .and. (.not. relax)) then
+        if (model_eqns == 3 ) then
+#ifdef _CRAYFTN
             call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
-        end if
+#else
+            if(.not. relax) then
+               call s_pressure_relaxation_procedure(q_cons_ts(2)%vf)
+           end if 
+#endif
+       end if
+
 
         if (ib) then
             if (qbmm .and. .not. polytropic) then
@@ -770,9 +799,16 @@ contains
 
         if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
 
-        if (model_eqns == 3 .and. (.not. relax)) then
+        if (model_eqns == 3 ) then
+#ifdef _CRAYFTN
             call s_pressure_relaxation_procedure(q_cons_ts(1)%vf)
-        end if
+#else
+            if(.not. relax) then
+               call s_pressure_relaxation_procedure(q_cons_ts(1)%vf)
+           end if 
+#endif
+       end if
+
 
         if (ib) then
             if (qbmm .and. .not. polytropic) then

diff --git a/src/simulation/m_weno.fpp b/src/simulation/m_weno.fpp
@@ -66,7 +66,6 @@ module m_weno
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :, :), poly_coef_cbR_z)
     !$acc declare link(poly_coef_cbL_x, poly_coef_cbL_y, poly_coef_cbL_z)
     !$acc declare link(poly_coef_cbR_x, poly_coef_cbR_y, poly_coef_cbR_z)
-    !$acc declare link(poly_coef_L, poly_coef_R)
 #else
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: poly_coef_cbL_x
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: poly_coef_cbL_y
@@ -76,8 +75,6 @@ module m_weno
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: poly_coef_cbR_y
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: poly_coef_cbR_z
 #endif
-    real(kind(0d0)), pointer, dimension(:, :, :) :: poly_coef_L
-    real(kind(0d0)), pointer, dimension(:, :, :) :: poly_coef_R
 
     !    real(kind(0d0)), pointer, dimension(:, :, :) :: poly_coef_L => null()
     !    real(kind(0d0)), pointer, dimension(:, :, :) :: poly_coef_R => null()
@@ -97,7 +94,7 @@ module m_weno
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :), d_cbR_x)
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :), d_cbR_y)
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :), d_cbR_z)
-    !$acc declare link(d_cbL_x, d_cbL_y, d_cbL_z, d_cbR_x, d_cbR_y, d_cbR_z, d_L, d_R)
+    !$acc declare link(d_cbL_x, d_cbL_y, d_cbL_z, d_cbR_x, d_cbR_y, d_cbR_z)
 #else
     real(kind(0d0)), target, allocatable, dimension(:, :) :: d_cbL_x
     real(kind(0d0)), target, allocatable, dimension(:, :) :: d_cbL_y
@@ -107,8 +104,6 @@ module m_weno
     real(kind(0d0)), target, allocatable, dimension(:, :) :: d_cbR_y
     real(kind(0d0)), target, allocatable, dimension(:, :) :: d_cbR_z
 #endif
-    real(kind(0d0)), pointer, dimension(:, :) :: d_L
-    real(kind(0d0)), pointer, dimension(:, :) :: d_R
 !    real(kind(0d0)), pointer, dimension(:, :) :: d_L => null()
 !    real(kind(0d0)), pointer, dimension(:, :) :: d_R => null()
     !> @}
@@ -122,13 +117,12 @@ module m_weno
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :, :), beta_coef_x)
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :, :), beta_coef_y)
     @:CRAY_DECLARE_GLOBAL(real(kind(0d0)), dimension(:, :, :), beta_coef_z)
-    !$acc declare link(beta_coef_x, beta_coef_y, beta_coef_z, beta_coef)
+    !$acc declare link(beta_coef_x, beta_coef_y, beta_coef_z)
 #else
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: beta_coef_x
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: beta_coef_y
     real(kind(0d0)), target, allocatable, dimension(:, :, :) :: beta_coef_z
 #endif
-    real(kind(0d0)), pointer, dimension(:, :, :) :: beta_coef
 !    real(kind(0d0)), pointer, dimension(:, :, :) :: beta_coef => null()
     !> @}
 
@@ -153,8 +147,7 @@ module m_weno
     !$acc                v_rs_ws_x, v_rs_ws_y, v_rs_ws_z, &
     !$acc                poly_coef_cbL_x,poly_coef_cbL_y,poly_coef_cbL_z, &
     !$acc                poly_coef_cbR_x,poly_coef_cbR_y,poly_coef_cbR_z,d_cbL_x,       &
-    !$acc                d_cbL_y,d_cbL_z,d_cbR_x,d_cbR_y,d_cbR_z,beta_coef_x,beta_coef_y,beta_coef_z,   &
-    !$acc                v_size, )
+    !$acc                d_cbL_y,d_cbL_z,d_cbR_x,d_cbR_y,d_cbR_z,beta_coef_x,beta_coef_y,beta_coef_z)
 #endif
 
 contains
@@ -741,6 +734,11 @@ contains
                     end do
                 end do
                 !$acc end parallel loop
+
+                if (mp_weno) then
+                    call s_preserve_monotonicity(v_rs_ws_${XYZ}$, vL_rs_vf_${XYZ}$, &
+                     vR_rs_vf_${XYZ}$)                   
+                end if
             end if
             #:endfor
         end if