Change folder structure for QLGYRO directory for GPU optimisation

gafusion · Sep 5, 2024 · 6d8e838 · 6d8e838
1 parent 5aa43d1
commit 6d8e838
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 22 deletions.
diff --git a/cgyro/src/cgyro_globals.F90 b/cgyro/src/cgyro_globals.F90
@@ -210,6 +210,8 @@ module cgyro_globals
   !
   integer :: nt1,nt2,nt_loc
   integer :: n_toroidal_procs
+  integer :: worker_index=0
+  integer :: iteration_method=0
   !---------------------------------------------------------------
 
   !---------------------------------------------------------------

diff --git a/qlgyro/src/qlgyro_cgyro_cleanup.F90 b/qlgyro/src/qlgyro_cgyro_cleanup.F90
@@ -76,8 +76,8 @@ subroutine qlgyro_cgyro_cleanup
         ftemp = ftemp / a_norm
 
         cgyro_wavefunction_out(i_tor, i_field, :) = reshape(ftemp, (/cgyro_n_radial_in * cgyro_n_theta_in/))
-  end do
-enddo
+     end do
+  enddo
 
   if (auto_box_size .eq. 0) then
      ! Get ballooning theta from out.cgyro.grids

diff --git a/qlgyro/src/qlgyro_cgyro_interface.f90 b/qlgyro/src/qlgyro_cgyro_interface.f90
@@ -202,6 +202,9 @@ module qlgyro_cgyro_interface
 
   logical :: cgyro_printout_in
 
+  integer :: cgyro_worker_index_in=0
+  integer :: cgyro_iteration_method_in=0
+
 contains
 
 
@@ -373,6 +376,9 @@ subroutine map_global2interface()
 
     cgyro_printout_in = printout
 
+    cgyro_worker_index_in = worker_index
+    cgyro_iteration_method_in = iteration_method
+
   end subroutine map_global2interface
 
   subroutine allocate_cgyro_interface
@@ -571,6 +577,9 @@ subroutine map_interface2global()
     signal = cgyro_signal_out
     error_status = cgyro_error_status_out
 
+    worker_index = cgyro_worker_index_in
+    iteration_method = cgyro_iteration_method_in
+
   end subroutine map_interface2global
 
 

diff --git a/qlgyro/src/qlgyro_globals.f90 b/qlgyro/src/qlgyro_globals.f90
@@ -304,7 +304,8 @@ module qlgyro_globals
   integer :: flux_method
   integer, dimension(:), allocatable :: flux_method_vec
   integer :: i_tran=1
-  character(13) :: iter_path="./           "
+  character(30) :: iter_path="./"
+  character(30) :: worker_str
   integer :: i_bc
   integer :: flux_counter
   integer :: i_ash

diff --git a/qlgyro/src/qlgyro_inout.f90 b/qlgyro/src/qlgyro_inout.f90
@@ -377,7 +377,7 @@ subroutine init_qlgyro_status
    CHARACTER(18) :: statusfile="out.qlgyro.status"
    CHARACTER(98) :: full_statusfile
 
-   full_statusfile = trim(path)//statusfile
+   full_statusfile = trim(path)//trim(iter_path)//statusfile
 
    call MPI_FILE_DELETE(full_statusfile, MPI_INFO_NULL, ierr)
 
@@ -429,7 +429,7 @@ subroutine write_qlgyro_status(runs, colors, i_kypx0, status, color_in)
    CHARACTER(98) :: full_statusfile
    CHARACTER(14) :: ky_status
 
-   full_statusfile = trim(path)//statusfile
+   full_statusfile = trim(path)//trim(iter_path)//statusfile
 
    offset = (i_kypx0 - 1) * statstr
 
@@ -485,7 +485,7 @@ subroutine get_set_qlgyro_status(i_kypx0, run_status, loop_cycle, runs, colors,
    CHARACTER(98) :: full_statusfile
    CHARACTER(14) :: ky_status
 
-   full_statusfile = trim(path)//statusfile
+   full_statusfile = trim(path)//trim(iter_path)//statusfile
 
    offset = (i_kypx0 - 1) * statstr + 44
 
@@ -540,7 +540,7 @@ subroutine get_qlgyro_status(i_kypx0, status, loop_cycle)
    CHARACTER(1) :: outstr
    CHARACTER(98) :: full_statusfile
 
-   full_statusfile = trim(path)//statusfile
+   full_statusfile = trim(path)//trim(iter_path)//statusfile
 
    offset = (i_kypx0-1) * statstr + 44
    call MPI_FILE_OPEN(qlgyro_comm, full_statusfile, &
@@ -579,7 +579,7 @@ subroutine read_qlgyro_status(runs, cores)
    INTEGER(kind=MPI_OFFSET_KIND) :: offset1, offset2
    CHARACTER(98) :: full_statusfile
 
-   full_statusfile = trim(path)//statusfile
+   full_statusfile = trim(path)//trim(iter_path)//statusfile
 
    call MPI_FILE_OPEN(qlgyro_comm, full_statusfile, &
         MPI_MODE_RDONLY, MPI_INFO_NULL, file_handle, ierr)

diff --git a/qlgyro/src/qlgyro_ky_spectrum.f90 b/qlgyro/src/qlgyro_ky_spectrum.f90
@@ -10,6 +10,12 @@ SUBROUTINE qlgyro_ky_spectrum(kymin)
       real, intent(in) :: kymin
       real, dimension(12) :: n_tor
       integer :: i_ky
+      integer :: mg_ky_grid=0
+
+      if (tglf_kygrid_model_in .eq. -1) then
+         kygrid_model_in = 0
+         mg_ky_grid = 1
+      end if
 
       call get_ky_spectrum
 
@@ -18,17 +24,17 @@ SUBROUTINE qlgyro_ky_spectrum(kymin)
          allocate(tglf_ky_spectrum_out(nky), tglf_dky_spectrum_out(nky))
       end if
 
-      ! if (tglf_kygrid_model_in .eq. 0) then
-
-      !    n_tor = (/2.0, 3.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 70.0, 100.0, 120.0, 140./)
-      !    ky_spectrum = 0.0
-      !    ky_spectrum(1) = kymin * n_tor(1)
-      !    dky_spectrum(1) = kymin * n_tor(1)
-      !    do i_ky=2,nky
-      !       ky_spectrum(i_ky) = kymin * n_tor(i_ky)
-      !       dky_spectrum(i_ky) = ky_spectrum(i_ky) - ky_spectrum(i_ky -1)
-      !    end do
-      ! end if
+      if (mg_ky_grid .eq. 1) then
+
+         n_tor = (/2.0, 3.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 70.0, 100.0, 120.0, 140./)
+         ky_spectrum = 0.0
+         ky_spectrum(1) = kymin * n_tor(1)
+         dky_spectrum(1) = kymin * n_tor(1)
+         do i_ky=2,nky
+            ky_spectrum(i_ky) = kymin * n_tor(i_ky)
+            dky_spectrum(i_ky) = ky_spectrum(i_ky) - ky_spectrum(i_ky -1)
+         end do
+      end if
 
       tglf_ky_spectrum_out = ky_spectrum(:nky)
       tglf_dky_spectrum_out = dky_spectrum(:nky)

diff --git a/qlgyro/src/qlgyro_run.f90 b/qlgyro/src/qlgyro_run.f90
@@ -60,6 +60,12 @@ subroutine qlgyro_run(lpath_in, qlgyro_comm_in, i_tran_in)
      end if
 
      write(iter_path, format_string) "ITERATION_", i_tran, "/"
+
+     ! Adjust path directory if running TGYRO in parallel
+     if (cgyro_iteration_method_in .eq. 5) then
+        write(worker_str, *) cgyro_worker_index_in
+        iter_path = trim(iter_path)//trim(adjustl(worker_str))//"/"
+     end if
      if (i_proc_global .eq. 0) then
         call system("mkdir -p "//trim(path)//trim(iter_path))
      end if

diff --git a/qlgyro/src/qlgyro_run_cgyro_balloon.f90 b/qlgyro/src/qlgyro_run_cgyro_balloon.f90
@@ -179,7 +179,7 @@ subroutine qlgyro_run_cgyro_balloon
      ! Creates directory for each ky and changes into it for that run
      write(kystr, fmt_str_ky) i_ky_local
      write(px0str, fmt_str_px0) i_px0_local
-     runpath = trim(trim(path)//trim(iter_path)//'/KY_'//trim(adjustl(kystr)))//"_PX0_"//trim(adjustl(px0str))//"/"
+     runpath = trim(trim(path)//trim(iter_path)//'KY_'//trim(adjustl(kystr)))//"_PX0_"//trim(adjustl(px0str))//"/"
 
      if (adjoint .eq. 0) then
         call system("mkdir -p "//runpath)

diff --git a/qlgyro/src/qlgyro_sum_fluxes.f90 b/qlgyro/src/qlgyro_sum_fluxes.f90
@@ -167,8 +167,10 @@ SUBROUTINE qlgyro_sum_fluxes
       end do
 
       if (i_proc_global == 0) then
-         write(*,*) 'QLGYRO Heat     fluxes (Qe, Qi): ', tglf_elec_eflux_out, tglf_ion_eflux_out(1)
-         write(*,*) 'QLGYRO Particle fluxes (Qe, Qi): ', tglf_elec_pflux_out, tglf_ion_pflux_out(1)
+         open(unit=1,file=trim(runfile),position='append')
+         write(1,*) 'QLGYRO Heat     fluxes (Qe, Qi): ', tglf_elec_eflux_out, tglf_ion_eflux_out(1)
+         write(1,*) 'QLGYRO Particle fluxes (Qe, Qi): ', tglf_elec_pflux_out, tglf_ion_pflux_out(1)
+         close(1)
       end if
 21 format(A35, F8.3, F8.3)
     END SUBROUTINE qlgyro_sum_fluxes

diff --git a/tgyro/src/tgyro_cgyro_map.f90 b/tgyro/src/tgyro_cgyro_map.f90
@@ -194,5 +194,7 @@ subroutine tgyro_cgyro_map
   cgyro_rho_star_norm_in = rho_s(i_r) / r_min
   cgyro_vth_norm_in = c_s(i_r)
 
+  cgyro_worker_index_in = worker_index
+  cgyro_iteration_method_in = tgyro_iteration_method
 
 end subroutine tgyro_cgyro_map