updates hip configuration

cenjinzhiyu · Apr 13, 2021 · a992fac · a992fac
1 parent bf125bb
commit a992fac
Show file tree

Hide file tree

Showing 6 changed files with 187 additions and 36 deletions.
diff --git a/Makefile.in b/Makefile.in
@@ -257,7 +257,19 @@ GENCODE_AMD_MI25 = --amdgpu-target=gfx900
 GENCODE_AMD_MI50 = --amdgpu-target=gfx906
 GENCODE_AMD_MI100 = --amdgpu-target=gfx908
 
-@COND_HIP_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI50) $(GENCODE_AMD_MI100)
+# default targets
+# AMD default MI50 & MI100
+@COND_HIP_TRUE@@COND_HIP_PLATFORM_AMD_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI50) $(GENCODE_AMD_MI100)
+@COND_HIP_TRUE@@COND_HIP_PLATFORM_AMD_TRUE@HIP_CFLAG_ENDING = -x hip
+# NVIDIA default Tesla
+@COND_HIP_TRUE@@COND_HIP_PLATFORM_NVIDIA_TRUE@GENCODE_HIP = $(GENCODE_30)
+@COND_HIP_TRUE@@COND_HIP_PLATFORM_AMD_TRUE@HIP_CFLAG_ENDING = -x cu
+
+# specific targets
+@COND_HIP_TRUE@@COND_HIP_MI8_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI8)    # --with-hip=MI8 ..
+@COND_HIP_TRUE@@COND_HIP_MI25_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI25)   # --with-hip=MI25 ..
+@COND_HIP_TRUE@@COND_HIP_MI50_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI50)   # --with-hip=MI50 ..
+@COND_HIP_TRUE@@COND_HIP_MI100_TRUE@GENCODE_HIP = $(GENCODE_AMD_MI100)  # --with-hip=MI100 ..
 
 HIP_FLAGS = @HIP_FLAGS@
 HIP_INC = @HIP_CPPFLAGS@ $(MPI_INCLUDES)
@@ -279,7 +291,7 @@ HIP_INC = @HIP_CPPFLAGS@ $(MPI_INCLUDES)
 #SET_MPI_LIB = ${MPI_LIB_PATH} $(shell echo ${MPI_LIBS} | sed -e 's/\b\([a-z]\+\)[ ,\n]\1/\1/g'|sed 's/[^ ]* */-l&/g')
 #FCLINK = $(HIPCC) $(SET_MPI_LIB)
 ## mpich
-#mpif90 -link_info
+# from: mpif90 -link_info
 #FCLINK = $(HIPCC) -L/usr/lib/x86_64-linux-gnu -lmpichfort -lmpich -lgfortran -lm -shared-libgcc
 
 # checks if any GPU flag set

diff --git a/configure b/configure
@@ -633,6 +633,10 @@ OMP_FCFLAGS
 HIP_LIBS
 HIP_LDFLAGS
 HIP_CPPFLAGS
+COND_HIP_PLATFORM_NVIDIA_FALSE
+COND_HIP_PLATFORM_NVIDIA_TRUE
+COND_HIP_PLATFORM_AMD_FALSE
+COND_HIP_PLATFORM_AMD_TRUE
 HIPCONFIG_PROG
 HIPCC_PROG
 HIP_LIB
@@ -718,6 +722,14 @@ COND_VTK_FALSE
 COND_VTK_TRUE
 COND_OMP_FALSE
 COND_OMP_TRUE
+COND_HIP_MI100_FALSE
+COND_HIP_MI100_TRUE
+COND_HIP_MI50_FALSE
+COND_HIP_MI50_TRUE
+COND_HIP_MI25_FALSE
+COND_HIP_MI25_TRUE
+COND_HIP_MI8_FALSE
+COND_HIP_MI8_TRUE
 COND_HIP_FALSE
 COND_HIP_TRUE
 COND_CUDA_PLUS_FALSE
@@ -2992,6 +3004,38 @@ else
   COND_HIP_FALSE=
 fi
 
+ if test x"$want_hip" = xMI8; then
+  COND_HIP_MI8_TRUE=
+  COND_HIP_MI8_FALSE='#'
+else
+  COND_HIP_MI8_TRUE='#'
+  COND_HIP_MI8_FALSE=
+fi
+
+ if test x"$want_hip" = xMI25; then
+  COND_HIP_MI25_TRUE=
+  COND_HIP_MI25_FALSE='#'
+else
+  COND_HIP_MI25_TRUE='#'
+  COND_HIP_MI25_FALSE=
+fi
+
+ if test x"$want_hip" = xMI50; then
+  COND_HIP_MI50_TRUE=
+  COND_HIP_MI50_FALSE='#'
+else
+  COND_HIP_MI50_TRUE='#'
+  COND_HIP_MI50_FALSE=
+fi
+
+ if test x"$want_hip" = xMI100; then
+  COND_HIP_MI100_TRUE=
+  COND_HIP_MI100_FALSE='#'
+else
+  COND_HIP_MI100_TRUE='#'
+  COND_HIP_MI100_FALSE=
+fi
+
 
 ###
 ### OPENMP
@@ -8391,39 +8435,75 @@ fi
 fi
     { $as_echo "$as_me:${as_lineno-$LINENO}: hipconfig program: $HIPCONFIG_PROG" >&5
 $as_echo "$as_me: hipconfig program: $HIPCONFIG_PROG" >&6;}
-    { $as_echo "$as_me:${as_lineno-$LINENO}: hip path: $HIP_PATH" >&5
-$as_echo "$as_me: hip path: $HIP_PATH" >&6;}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: HIP path: $HIP_PATH" >&5
+$as_echo "$as_me: HIP path: $HIP_PATH" >&6;}
 
     #AS_IF([test x"$HIP_INC" = x],[
-    #    HIP_INC="$HIP_PATH/include"
-    #])
-    #AS_IF([test x"$HIP_LIB" = x],[
-    #    HIP_LIB="$HIP_PATH/lib"
+    #    HIP_INC="${HIP_PATH}/include"
     #])
 
+    # adds default HIP library
+    if test x"$HIP_LIB" = x -a x"${HIP_PATH}" != x; then :
+
+        HIP_LIB="${HIP_PATH}/lib"
+        { $as_echo "$as_me:${as_lineno-$LINENO}: adding default HIP library path: ${HIP_LIB}" >&5
+$as_echo "$as_me: adding default HIP library path: ${HIP_LIB}" >&6;}
+
+fi
+
     # checks platform
     if test x"$HIPCONFIG_PROG" != x ; then
       GPU_PLATFORM=`$HIPCONFIG_PROG --platform`
-      if test x"$GPU_PLATFORM" = xnvcc; then :
+      if test x"$GPU_PLATFORM" = xnvidia; then :
 
-          { $as_echo "$as_me:${as_lineno-$LINENO}: HIP PLATFORM NVCC detected." >&5
-$as_echo "$as_me: HIP PLATFORM NVCC detected." >&6;}
+          { $as_echo "$as_me:${as_lineno-$LINENO}: HIP PLATFORM NVIDIA detected." >&5
+$as_echo "$as_me: HIP PLATFORM NVIDIA detected." >&6;}
+          HIP_PLATFORM=nvcc
           FLAG_PLATFORM="${FC_DEFINE}__HIP_PLATFORM_NVCC__"
 
 fi
-      if test x"$GPU_PLATFORM" = xhcc; then :
 
-          { $as_echo "$as_me:${as_lineno-$LINENO}: HIP PLATFORM HCC detected." >&5
-$as_echo "$as_me: HIP PLATFORM HCC detected." >&6;}
+      if test x"$GPU_PLATFORM" = xamd; then :
+
+          { $as_echo "$as_me:${as_lineno-$LINENO}: HIP PLATFORM AMD detected." >&5
+$as_echo "$as_me: HIP PLATFORM AMD detected." >&6;}
+          HIP_PLATFORM=hcc
           FLAG_PLATFORM="${FC_DEFINE}__HIP_PLATFORM_HCC__"
 
 fi
       # compiler might require a platform selection flag
-      if [ "$HIP_FLAGS" != *"HIP_PLATFORM"* ]; then
-        # adds PLATFORM flag
-        HIP_FLAGS="${HIP_FLAGS} ${FLAG_PLATFORM}"
-      fi
+      case "${HIP_FLAGS}" in
+        *HIP_PLATFORM*) { $as_echo "$as_me:${as_lineno-$LINENO}: hip flags contain platform specifier: ${HIP_FLAGS}" >&5
+$as_echo "$as_me: hip flags contain platform specifier: ${HIP_FLAGS}" >&6;} ;;
+        *) { $as_echo "$as_me:${as_lineno-$LINENO}: adding flag for platform specifier ${FLAG_PLATFORM}" >&5
+$as_echo "$as_me: adding flag for platform specifier ${FLAG_PLATFORM}" >&6;};
+           HIP_FLAGS="${HIP_FLAGS} ${FLAG_PLATFORM}"
+           { $as_echo "$as_me:${as_lineno-$LINENO}: HIP_FLAGS: ${HIP_FLAGS}" >&5
+$as_echo "$as_me: HIP_FLAGS: ${HIP_FLAGS}" >&6;} ;;
+      esac
     fi
+    # sets default platform to AMD if not set
+    if test x"$GPU_PLATFORM" = x; then :
+
+      GPU_PLATFORM=amd
+
+fi
+     if test x"$GPU_PLATFORM" = xamd; then
+  COND_HIP_PLATFORM_AMD_TRUE=
+  COND_HIP_PLATFORM_AMD_FALSE='#'
+else
+  COND_HIP_PLATFORM_AMD_TRUE='#'
+  COND_HIP_PLATFORM_AMD_FALSE=
+fi
+
+     if test x"$GPU_PLATFORM" = xnvidia; then
+  COND_HIP_PLATFORM_NVIDIA_TRUE=
+  COND_HIP_PLATFORM_NVIDIA_FALSE='#'
+else
+  COND_HIP_PLATFORM_NVIDIA_TRUE='#'
+  COND_HIP_PLATFORM_NVIDIA_FALSE=
+fi
+
 
     # for compilation errors like:
     #   /usr/bin/ld: obj/assemble_MPI_scalar_gpu.hip.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used
@@ -8432,6 +8512,13 @@ fi
     #   HIP_FLAGS="-fPIC"
     # or
     #   HIP_FLAGS="-fno-gpu-rdc"
+    # compiler might require a -fPIC flag
+    case "${HIP_FLAGS}" in
+      *fPIC*) { $as_echo "$as_me:${as_lineno-$LINENO}: HIP flags contain position independent code flag -fPIC: ${HIP_FLAGS}" >&5
+$as_echo "$as_me: HIP flags contain position independent code flag -fPIC: ${HIP_FLAGS}" >&6;};;
+      *) { $as_echo "$as_me:${as_lineno-$LINENO}: consider adding compilation flag for position independent code: HIP_FLAGS=\"-fPIC\"" >&5
+$as_echo "$as_me: consider adding compilation flag for position independent code: HIP_FLAGS=\"-fPIC\"" >&6;};;
+    esac
 
     # test lib & include
     if test x"$HIP_LIB" != x; then :
@@ -10738,6 +10825,22 @@ if test -z "${COND_HIP_TRUE}" && test -z "${COND_HIP_FALSE}"; then
   as_fn_error $? "conditional \"COND_HIP\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${COND_HIP_MI8_TRUE}" && test -z "${COND_HIP_MI8_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_MI8\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${COND_HIP_MI25_TRUE}" && test -z "${COND_HIP_MI25_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_MI25\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${COND_HIP_MI50_TRUE}" && test -z "${COND_HIP_MI50_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_MI50\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${COND_HIP_MI100_TRUE}" && test -z "${COND_HIP_MI100_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_MI100\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 if test -z "${COND_OMP_TRUE}" && test -z "${COND_OMP_FALSE}"; then
   as_fn_error $? "conditional \"COND_OMP\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -10768,6 +10871,14 @@ esac
 case $FC_MODOUT in #(
   *\ ) FC_MODOUT=$FC_MODOUT'${ac_empty}' ;;
 esac
+if test -z "${COND_HIP_PLATFORM_AMD_TRUE}" && test -z "${COND_HIP_PLATFORM_AMD_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_PLATFORM_AMD\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${COND_HIP_PLATFORM_NVIDIA_TRUE}" && test -z "${COND_HIP_PLATFORM_NVIDIA_FALSE}"; then
+  as_fn_error $? "conditional \"COND_HIP_PLATFORM_NVIDIA\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 
 : "${CONFIG_STATUS=./config.status}"
 ac_write_fail=0

diff --git a/configure.ac b/configure.ac
@@ -135,6 +135,10 @@ AC_ARG_WITH([hip],
     [want_hip="$withval"],
     [want_hip=no])
 AM_CONDITIONAL([COND_HIP], [test x"$want_hip" != xno])
+AM_CONDITIONAL([COND_HIP_MI8], [test x"$want_hip" = xMI8])
+AM_CONDITIONAL([COND_HIP_MI25], [test x"$want_hip" = xMI25])
+AM_CONDITIONAL([COND_HIP_MI50], [test x"$want_hip" = xMI50])
+AM_CONDITIONAL([COND_HIP_MI100], [test x"$want_hip" = xMI100])
 
 ###
 ### OPENMP
@@ -520,32 +524,46 @@ AS_IF([test x"$want_hip" != xno], [
         HIP_PATH=`$HIPCONFIG_PROG --path`
     ])
     AC_MSG_NOTICE([hipconfig program: $HIPCONFIG_PROG])
-    AC_MSG_NOTICE([hip path: $HIP_PATH])
+    AC_MSG_NOTICE([HIP path: $HIP_PATH])
 
     #AS_IF([test x"$HIP_INC" = x],[
-    #    HIP_INC="$HIP_PATH/include"
-    #])
-    #AS_IF([test x"$HIP_LIB" = x],[
-    #    HIP_LIB="$HIP_PATH/lib"
+    #    HIP_INC="${HIP_PATH}/include"
     #])
 
+    # adds default HIP library
+    AS_IF([test x"$HIP_LIB" = x -a x"${HIP_PATH}" != x],[
+        HIP_LIB="${HIP_PATH}/lib"
+        AC_MSG_NOTICE([adding default HIP library path: ${HIP_LIB}])
+    ])
+
     # checks platform
     if test x"$HIPCONFIG_PROG" != x ; then
       GPU_PLATFORM=`$HIPCONFIG_PROG --platform`
-      AS_IF([test x"$GPU_PLATFORM" = xnvcc], [
-          AC_MSG_NOTICE([HIP PLATFORM NVCC detected.])
+      AS_IF([test x"$GPU_PLATFORM" = xnvidia], [
+          AC_MSG_NOTICE([HIP PLATFORM NVIDIA detected.])
+          HIP_PLATFORM=nvcc
           FLAG_PLATFORM="${FC_DEFINE}__HIP_PLATFORM_NVCC__"
       ])
-      AS_IF([test x"$GPU_PLATFORM" = xhcc], [
-          AC_MSG_NOTICE([HIP PLATFORM HCC detected.])
+
+      AS_IF([test x"$GPU_PLATFORM" = xamd], [
+          AC_MSG_NOTICE([HIP PLATFORM AMD detected.])
+          HIP_PLATFORM=hcc
           FLAG_PLATFORM="${FC_DEFINE}__HIP_PLATFORM_HCC__"
       ])
       # compiler might require a platform selection flag
-      if [[ "$HIP_FLAGS" != *"HIP_PLATFORM"* ]]; then
-        # adds PLATFORM flag
-        HIP_FLAGS="${HIP_FLAGS} ${FLAG_PLATFORM}"
-      fi
+      case "${HIP_FLAGS}" in
+        *HIP_PLATFORM*) AC_MSG_NOTICE([hip flags contain platform specifier: ${HIP_FLAGS}]) ;;
+        *) AC_MSG_NOTICE([adding flag for platform specifier ${FLAG_PLATFORM}]);
+           HIP_FLAGS="${HIP_FLAGS} ${FLAG_PLATFORM}"
+           AC_MSG_NOTICE([HIP_FLAGS: ${HIP_FLAGS}]) ;;
+      esac
     fi
+    # sets default platform to AMD if not set
+    AS_IF([test x"$GPU_PLATFORM" = x],[
+      GPU_PLATFORM=amd
+    ])
+    AM_CONDITIONAL([COND_HIP_PLATFORM_AMD], [test x"$GPU_PLATFORM" = xamd])
+    AM_CONDITIONAL([COND_HIP_PLATFORM_NVIDIA], [test x"$GPU_PLATFORM" = xnvidia])
 
     # for compilation errors like:
     #   /usr/bin/ld: obj/assemble_MPI_scalar_gpu.hip.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used
@@ -554,6 +572,11 @@ AS_IF([test x"$want_hip" != xno], [
     #   HIP_FLAGS="-fPIC"
     # or
     #   HIP_FLAGS="-fno-gpu-rdc"
+    # compiler might require a -fPIC flag
+    case "${HIP_FLAGS}" in
+      *fPIC*) AC_MSG_NOTICE([HIP flags contain position independent code flag -fPIC: ${HIP_FLAGS}]);;
+      *) AC_MSG_NOTICE([consider adding compilation flag for position independent code: HIP_FLAGS="-fPIC"]);;
+    esac
 
     # test lib & include
     AS_IF([test x"$HIP_LIB" != x],[

diff --git a/src/gpu/mesh_constants_gpu.h b/src/gpu/mesh_constants_gpu.h
@@ -625,9 +625,10 @@ typedef struct mesh_ {
   realw* h_send_b_accel_buffer;
 
   //realw* send_buffer;
+  //realw* recv_buffer;
+
   realw* h_recv_accel_buffer;
   realw* h_recv_b_accel_buffer;
-  realw* recv_buffer;
 
   int size_mpi_buffer;
   int size_mpi_buffer_potential;

diff --git a/src/gpu/prepare_mesh_constants_cuda.cu b/src/gpu/prepare_mesh_constants_cuda.cu
@@ -517,6 +517,7 @@ void FC_FUNC_(prepare_fields_acoustic_device,
     if (mp->simulation_type == 3 || ( mp->simulation_type == 1 && mp->save_forward )){
       // note: b_reclen_potential is record length in bytes ( CUSTOM_REAL * NGLLSQUARE * num_abs_boundary_faces )
       mp->d_b_reclen_potential = *b_reclen_potential;
+
       //daniel todo: reclen includes sizeof(realw)
       gpuMalloc_realw((void**)&mp->d_b_absorb_potential,mp->d_b_reclen_potential/sizeof(realw));
       gpuMemcpy_todevice_realw(mp->d_b_absorb_potential,b_absorb_potential,mp->d_b_reclen_potential/sizeof(realw));
@@ -730,15 +731,18 @@ void FC_FUNC_(prepare_fields_elastic_device,
 
     // send buffer
     gpuMallocHost_realw((void**)&(mp->h_send_accel_buffer),mp->size_mpi_buffer);
-    //mp->send_buffer = (float*)malloc((mp->size_mpi_buffer)*sizeof(float));
+
+    // unused so far..
+    //mp->send_buffer = (realw*)malloc((mp->size_mpi_buffer)*sizeof(realw));
     // extra buffer for adjoint, not needed so far..., can use the same buffer for both forward/adjoint mpi exchanges
     //gpuMallocHost_realw((void**)&(mp->h_send_b_accel_buffer),mp->size_mpi_buffer);
-    //mp->b_send_buffer = (float*)malloc((size_mpi_buffer)*sizeof(float));
+    //mp->b_send_buffer = (realw*)malloc((size_mpi_buffer)*sizeof(realw));
 
     // receive buffer
     gpuMallocHost_realw((void**)&(mp->h_recv_accel_buffer),mp->size_mpi_buffer);
 
-    mp->recv_buffer = (realw*)malloc((mp->size_mpi_buffer)*sizeof(realw));
+    // unused so far..
+    //mp->recv_buffer = (realw*) malloc((mp->size_mpi_buffer)*sizeof(realw));
 
     // non-pinned buffer
     gpuMalloc_realw((void**)&(mp->d_send_accel_buffer),mp->size_mpi_buffer);

diff --git a/src/gpu/rules.mk b/src/gpu/rules.mk
@@ -144,7 +144,7 @@ ifeq ($(CUDA),yes)
 endif
 
 ## HIP compilation
-HIPCC_CFLAGS := ${HIP_CFLAGS} -x hip
+HIPCC_CFLAGS := ${HIP_CFLAGS} ${HIP_CFLAG_ENDING}  # adding either -x hip or -x cu depending on platform
 ifeq ($(HIP), yes)
   BUILD_VERSION_TXT += HIP
   SELECTOR_CFLAG += $(FC_DEFINE)USE_HIP