Branch of fesom2.6 including recom and tracer parallelisation

src/CMakeLists.txt

@@ -73,6 +73,9 @@ if(ENABLE_OPENMP)
    find_package(OpenMP REQUIRED COMPONENTS Fortran)
 endif()
 
+option(RECOM_COUPLED "Use RECOM" OFF)
+message(STATUS "RECOM_COUPLED: ${RECOM_COUPLED}")
+
 option(USE_ICEPACK "Use ICEPACK" OFF)
 message(STATUS "USE_ICEPACK: ${USE_ICEPACK}")
 
@@ -234,7 +237,9 @@ if(OPENMP_REPRODUCIBLE)
 endif()
 
 if(${RECOM_COUPLED})
-   target_compile_definitions(${PROJECT_NAME} PRIVATE __recom USE_PRECISION=2 __3Zoo2Det __coccos)# __usetp)
+#   target_compile_definitions(${PROJECT_NAME} PRIVATE __recom USE_PRECISION=2 __usetp)
+   target_compile_definitions(${PROJECT_NAME} PRIVATE __recom USE_PRECISION=2)
+#   target_compile_definitions(${PROJECT_NAME} PRIVATE __recom USE_PRECISION=2 __3Zoo2Det __coccos __usetp)
 endif()
 
 if(${CISO_COUPLED})

src/MOD_PARTIT.F90

@@ -71,6 +71,14 @@ module MOD_PARTIT
   integer              :: MPI_COMM_FESOM ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD)
   integer              :: MPI_COMM_WORLD ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD)
 
+#if defined(__recom) && defined(__usetp)
+! kh 11.11.21 communicator for multi FESOM group loop parallelization
+ integer               :: MPI_COMM_FESOM_WORLD
+
+! kh 17.11.21 communicator for multi FESOM group loop parallelization
+ integer               :: MPI_COMM_FESOM_SAME_RANK_IN_GROUPS
+#endif
+
   ! MPI Datatypes for interface exchange
   ! Element fields (2D; 2D integer; 3D with nl-1 or nl levels, 1 - 4 values)
   !                 small halo and / or full halo
@@ -87,6 +95,11 @@ module MOD_PARTIT
   integer, allocatable       :: s_mpitype_nod3D(:,:,:), r_mpitype_nod3D(:,:,:)
 
   integer            :: MPIERR
+
+#if defined(__recom) && defined(__usetp)
+! kh 11.11.21 multi FESOM group loop parallelization
+  integer            :: my_fesom_group
+#endif
 
   !!! remPtr_* are constructed during the runtime and shall not be dumped!!!
   integer, allocatable ::  remPtr_nod2D(:),  remList_nod2D(:)

src/MOD_TRACER.F90

@@ -20,6 +20,9 @@ MODULE MOD_TRACER
 real(kind=WP)                                 :: tra_adv_pv  = 1.  ! a parameter to be used in horizontal advection (for QR4C  it is the fraction of fourth-order contribution in the solution)
 integer                                       :: AB_order=2
 integer                                       :: ID
+!___________________________________________________________________________
+! TODO: Make it as a part of namelist.tra
+logical                                      :: ltra_diag       = .true. ! OG - tra_diag
 
 contains
 procedure WRITE_T_TRACER_DATA
@@ -41,7 +44,14 @@ MODULE MOD_TRACER
 !                              compute Tstar = 0.5*( T^(n+1) + T^n)
 real(kind=WP), allocatable, dimension(:,:,:)  :: dvd_trflx_hor, dvd_trflx_ver
 
-!_______________________________________________________________________________
+! in case ltra_diag=.true. --> calculate tracer diags ! OG - tra_diag
+real(kind=WP), allocatable                    :: tra_advhoriz(:,:,:), tra_advvert(:,:,:)
+real(kind=WP), allocatable                    :: tra_diff_part_hor_redi(:,:,:)
+real(kind=WP), allocatable                    :: tra_diff_part_ver_expl(:,:,:)
+real(kind=WP), allocatable                    :: tra_diff_part_ver_redi_expl(:,:,:)
+real(kind=WP), allocatable                    :: tra_diff_part_ver_impl(:,:,:)
+real(kind=WP), allocatable                    :: tra_recom_sms(:,:,:)
+
 ! The fct part
 real(kind=WP),allocatable,dimension(:,:)      :: fct_LO          ! Low-order solution
 real(kind=WP),allocatable,dimension(:,:)      :: adv_flux_hor    ! Antidif. horiz. contrib. from edges / backup for iterafive fct scheme

src/associate_part_ass.h

@@ -1,3 +1,7 @@
+!DIR$ if defined(__recom) .AND. defined(__usetp)
+MPI_COMM_FESOM_WORLD                => partit%MPI_COMM_FESOM_WORLD
+MPI_COMM_FESOM_SAME_RANK_IN_GROUPS  => partit%MPI_COMM_FESOM_SAME_RANK_IN_GROUPS
+!DIR$ endif
 MPI_COMM_FESOM  => partit%MPI_COMM_FESOM
 MPI_COMM_FESOM_IB       => partit%MPI_COMM_FESOM_IB
 com_nod2D       => partit%com_nod2D
@@ -13,9 +17,12 @@ eDim_edge2D     => partit%eDim_edge2D
 pe_status       => partit%pe_status
 elem_full_flag  => partit%elem_full_flag
 MPIERR          => partit%MPIERR
-MPIERR_IB               => partit%MPIERR_IB
+MPIERR_IB       => partit%MPIERR_IB
 npes            => partit%npes
 mype            => partit%mype
+!DIR$ if defined(__recom) .AND. defined(__usetp)
+my_fesom_group  => my_fesom_group
+!DIR$ endif
 maxPEnum        => partit%maxPEnum
 part            => partit%part
 

src/associate_part_def.h

@@ -1,3 +1,7 @@
+!DIR$ if defined(__recom) .AND. defined(__usetp)
+  integer,          pointer     :: MPI_COMM_FESOM_WORLD
+  integer,          pointer     :: MPI_COMM_FESOM_SAME_RANK_IN_GROUPS
+!DIR$ endif
   integer,          pointer     :: MPI_COMM_FESOM       ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD)
   integer,          pointer     :: MPI_COMM_FESOM_IB    ! FESOM communicator copy for icebergs LA: 2023-05-22
   type(com_struct), pointer     :: com_nod2D
@@ -20,6 +24,9 @@
   integer, pointer   :: MPIERR_IB       ! copy for icebergs LA: 2023-05-22
   integer, pointer   :: npes
   integer, pointer   :: mype
+!DIR$ if defined(__recom) .AND. defined(__usetp)
+  integer, pointer   :: my_fesom_group
+!DIR$ endif
   integer, pointer   :: maxPEnum
 
   integer, dimension(:), pointer     :: part

src/cpl_driver.F90

@@ -14,6 +14,9 @@ module cpl_driver
   !
   use mod_oasis                    ! oasis module
   use g_config, only : dt, use_icebergs, lwiso
+#if defined(__recom) && defined(__usetp)
+  use g_config, only : num_fesom_groups 
+#endif
   use o_param,  only : rad
   USE MOD_PARTIT
   implicit none
@@ -310,7 +313,12 @@ subroutine node_contours(my_x_corners, my_y_corners, partit, mesh)
     my_y_corners=my_y_corners/rad
     end subroutine node_contours
 
-  subroutine cpl_oasis3mct_init(partit, localCommunicator )
+#if defined(__recom) && defined(__usetp)
+  subroutine cpl_oasis3mct_init(partit, localCommunicator, num_fesom_groups)
+#else
+  subroutine cpl_oasis3mct_init(partit, localCommunicator)
+#endif
+
     USE MOD_PARTIT
     implicit none   
     save
@@ -324,6 +332,9 @@ subroutine cpl_oasis3mct_init(partit, localCommunicator )
     !
     integer, intent(OUT)       :: localCommunicator
     type(t_partit), intent(inout), target :: partit
+#if defined(__recom) && defined(__usetp)
+    integer, intent(inout)     :: num_fesom_groups
+#endif
     !
     ! Local declarations
     !
@@ -345,7 +356,11 @@ subroutine cpl_oasis3mct_init(partit, localCommunicator )
     !------------------------------------------------------------------
     ! 1st Initialize the OASIS3-MCT coupling system for the application
     !------------------------------------------------------------------
+#if defined(__recom) && defined(__usetp)
+    CALL oasis_init_comp(comp_id, comp_name, ierror, num_program_groups = num_fesom_groups)
+#else
     CALL oasis_init_comp(comp_id, comp_name, ierror )
+#endif
     IF (ierror /= 0) THEN
         CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'Init_comp failed.')
     ENDIF
@@ -356,7 +371,11 @@ subroutine cpl_oasis3mct_init(partit, localCommunicator )
         CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'comm_rank failed.')
     ENDIF
 
+#if defined(__recom) && defined(__usetp)
+    CALL oasis_get_localcomm_all_groups( localCommunicator, ierror )
+#else
     CALL oasis_get_localcomm( localCommunicator, ierror )
+#endif
     IF (ierror /= 0) THEN
         CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'get_local_comm failed.')
     ENDIF
@@ -606,6 +625,10 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh)
       print *, 'FESOM after Barrier'
     endif
 
+#if defined(__recom) && defined(__usetp)
+    if(partit%my_fesom_group == 0) then
+#endif
+
     if (mype .eq. localroot) then
       print *, 'FESOM before grid writing to oasis grid files'
        CALL oasis_start_grids_writing(il_flag)
@@ -632,6 +655,9 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh)
       print *, 'FESOM after terminate_grids_writing'
     endif !localroot
 
+#if defined(__recom) && defined(__usetp)
+    end if !(partit%my_fesom_group == 0) then     
+#endif
 
 
     DEALLOCATE(all_x_coords, all_y_coords, my_x_coords, my_y_coords, displs_from_all_pes, counts_from_all_pes)
@@ -900,15 +926,49 @@ subroutine cpl_oasis3mct_recv(ind, data_array, action, partit)
     endif    
 #endif
 
+#if defined(__recom) && defined(__usetp)
+! the coupling is in principle as it was before, i.e. the fesom processes - in group 0 - receive their data from echam
+    if(partit%my_fesom_group == 0) then
+#endif
+
     call oasis_get(recv_id(ind), seconds_til_now, exfld,info)
+
+#if defined(__recom) && defined(__usetp)
+    else
+
+! defensive: assignment statement "action=(info==3 ..." below is "don't care" in this case, because the actual value for action
+! is received via MPI_Bcast anyway
+        info = 0
+
+    end if
+#endif
+
     t2=MPI_Wtime()
  !
  ! FESOM's interpolation routine interpolates structured
  ! VarStrLoc coming from OASIS3MCT to local unstructured data_array
  ! and delivered back to FESOM.
    action=(info==3 .OR. info==10 .OR. info==11 .OR. info==12 .OR. info==13)
+
+#if defined(__recom) && defined(__usetp)
+   if(num_fesom_groups > 1) then
+      call MPI_Bcast(action, 1, MPI_LOGICAL, 0, partit%MPI_COMM_FESOM_SAME_RANK_IN_GROUPS, partit%MPIerr)
+   end if
+#endif 
+
    if (action) then
+#if defined(__recom) && defined(__usetp)
+      if(partit%my_fesom_group == 0) then
+#endif
       data_array(1:partit%myDim_nod2d) = exfld
+#if defined(__recom) && defined(__usetp)
+      end if
+
+      if(num_fesom_groups > 1) then
+          call MPI_Bcast(data_array, partit%myDim_nod2d, MPI_DOUBLE_PRECISION, 0, partit%MPI_COMM_FESOM_SAME_RANK_IN_GROUPS, partit%MPIerr)
+      end if
+#endif
+
       call exchange_nod(data_array, partit)
    end if   
    t3=MPI_Wtime()