From d7b8b060c9ff0cdb1483ec98c1ceb6c76c868b81 Mon Sep 17 00:00:00 2001 From: Boian Petkantchin Date: Tue, 7 May 2024 10:48:03 -0500 Subject: [PATCH] Refactor and add RCCL This drives adding hipcc, hipify, rocm-smi and rocminfo. hipcc is very particular where things like bitcodes and rocm_agent_enumerator should be. This necessitates merging the dynamic and dev/compiler staging directories. We need to run hipcc from the staging directory during building of rccl. Move rocm-cmake as separate project. LLVM references it. Make sure that subprojects do not reference other rocm stuff in default paths like /opt/rocm. This seems to be generally done by setting ROCM_PATH. We must the cautious if adding more parts of rocm to avoid contamination with existing installations. Use our repo tool manifest url as a default. When fetching sources do not set LLVM to the amd-staging, but let the manifest handle it. --- CMakeLists.txt | 183 +++++++++++++++--- README.md | 2 +- build_tools/apply_patches.sh | 6 + build_tools/fetch_sources.py | 24 ++- cmake/custom_install_amdgpu_compiler.cmake | 14 +- cmake/custom_install_amdgpu_runtime.cmake | 9 +- cmake/custom_install_amdgpu_runtime_dev.cmake | 27 ++- components/amd-llvm/CMakeLists.txt | 16 ++ components/rocm-core-libs/CMakeLists.txt | 3 - ...fy-install-headers-in-include-hipify.patch | 39 ++++ patches/rccl-overwrite-generated-files.patch | 20 ++ tests/dlopen-hip.c | 10 +- 12 files changed, 285 insertions(+), 68 deletions(-) create mode 100644 patches/hipify-install-headers-in-include-hipify.patch create mode 100644 patches/rccl-overwrite-generated-files.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index e41983e..9b11e4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,10 +25,12 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) endif() set(ROCM_MAJOR_VERSION 6) -set(ROCM_MINOR_VERSION 0) +set(ROCM_MINOR_VERSION 1) set(ROCM_PATCH_VERSION 0) -set(ROCM_VERSION "${ROCM_MAJOR_VERSION}.0.0" CACHE STRING "ROCM version") -set(AMDGPU_TARGETS "gfx90a gfx942 gfx1100" CACHE STRING "AMDGPU Targets") +set(ROCM_VERSION + "${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}" + CACHE STRING "ROCM version") +set(AMDGPU_TARGETS "gfx90a gfx940 gfx942 gfx1100" CACHE STRING "AMDGPU Targets") ################################################################################ # Global setup @@ -45,7 +47,9 @@ set(STAGING_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/staging_install") # External project setup ################################################################################ -set(ALWAYS_BUILD_SUBPROJECTS OFF) +option(ALWAYS_BUILD_SUBPROJECTS + "Don't let the brittle CMake external project machinery decide if a sub-project needs to rebuild. Always run the underlying build." + ON) set(FIND_PACKAGE_OPTIONS) macro(add_package_path PackageName path) @@ -63,6 +67,8 @@ set(DEFAULT_CMAKE_ARGS -DROCM_MINOR_VERSION=${ROCM_MINOR_VERSION} -DROCM_PATCH_VERSION=${ROCM_PATCH_VERSION} -DROCM_VERSION=${ROCM_VERSION} + "-DROCM_PATH=${STAGING_INSTALL_DIR}" + "-DCPACK_PACKAGING_INSTALL_PREFIX=${STAGING_INSTALL_DIR}" "-DAMDGPU_TARGETS=${AMDGPU_TARGETS}" -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER} @@ -78,6 +84,24 @@ if(CMAKE_CXX_VISIBILITY_PRESET) list(APPEND DEFAULT_CMAKE_ARGS ${CMAKE_CXX_VISIBILITY_PRESET}) endif() +################################################################################ +# rocm-cmake +################################################################################ + +ExternalProject_Add( + rocm-cmake + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-cmake + SOURCE_DIR "${ROCM_GIT_DIR}/rocm-cmake" + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) + +add_package_path(ROCM "${STAGING_INSTALL_DIR}/share/rocm/cmake") ################################################################################ # LLVM @@ -87,12 +111,12 @@ ExternalProject_Add( amd-llvm PREFIX ${CMAKE_CURRENT_BINARY_DIR}/amd-llvm SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/components/amd-llvm" + DEPENDS + rocm-cmake CMAKE_ARGS ${DEFAULT_CMAKE_ARGS} ${FIND_PACKAGE_OPTIONS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} - -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER} # We install llvm in its own sub-directory. -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/llvm USES_TERMINAL_CONFIGURE TRUE @@ -114,19 +138,63 @@ ExternalProject_Add( rocm-core-libs DEPENDS amd-llvm + rocm-cmake PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-core-libs SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/components/rocm-core-libs" CMAKE_ARGS ${DEFAULT_CMAKE_ARGS} ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/runtime_dynamic + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} USES_TERMINAL_CONFIGURE TRUE USES_TERMINAL_BUILD TRUE BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} ) -add_package_path(hsakmt "${STAGING_INSTALL_DIR}/runtime_dynamic/lib/cmake/hsakmt") -add_package_path(hsa-runtime64 "${STAGING_INSTALL_DIR}/runtime_dynamic/lib/cmake/hsa-runtime64") +add_package_path(hsakmt "${STAGING_INSTALL_DIR}/lib/cmake/hsakmt") +add_package_path(hsa-runtime64 "${STAGING_INSTALL_DIR}/lib/cmake/hsa-runtime64") + +################################################################################ +# rocminfo +################################################################################ + +ExternalProject_Add( + rocminfo + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocminfo + SOURCE_DIR "${ROCM_GIT_DIR}/rocminfo" + DEPENDS + rocm-core-libs + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) + +################################################################################ +# HIPCC +################################################################################ + +ExternalProject_Add( + hipcc + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/hipcc + # Can't build directly as part of LLVM, because we want hipcc staged under + # bin, not llvm/bin. + SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/hipcc" + DEPENDS + amd-llvm # runtime + rocminfo # runtime + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + # We install llvm in its own sub-directory. + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) ################################################################################ # CLR @@ -138,20 +206,19 @@ ExternalProject_Add( SOURCE_DIR "${ROCM_GIT_DIR}/clr" DEPENDS amd-llvm + hipcc rocm-core-libs - BUILD_ALWAYS TRUE CMAKE_ARGS ${DEFAULT_CMAKE_ARGS} ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/runtime_dynamic + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} -DHIP_PLATFORM=amd "-DOFFLOAD_ARCH_STR=${AMDGPU_TARGETS}" "-DHIP_CLANG_PATH=${STAGING_INSTALL_DIR}/llvm/bin" # Some junk needs this to be defined but is special cased so if empty, # bad things don't happen. - "-DHIPCC_BIN_DIR=" + "-DHIPCC_BIN_DIR=${STAGING_INSTALL_DIR}/bin" "-DHIP_COMMON_DIR=${ROCM_GIT_DIR}/HIP" - "-DROCM_PATH=${STAGING_INSTALL_DIR}/runtime_dynamic" # What is this? -DROCM_PATCH_VERSION=99999 -D__HIP_ENABLE_PCH=OFF @@ -160,12 +227,78 @@ ExternalProject_Add( "-D_STAMP=${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}" USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD ${ALWAYS_BUILD_SUBPROJECTS} + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} ) -add_package_path(HIP "${STAGING_INSTALL_DIR}/runtime_dynamic/lib/cmake/hip") -add_package_path(hip-lang "${STAGING_INSTALL_DIR}/runtime_dynamic/lib/cmake/hip-lang") -add_package_path(hiprtc "${STAGING_INSTALL_DIR}/runtime_dynamic/lib/cmake/hiprtc") +add_package_path(HIP "${STAGING_INSTALL_DIR}/lib/cmake/hip") +add_package_path(hip-lang "${STAGING_INSTALL_DIR}/lib/cmake/hip-lang") +add_package_path(hiprtc "${STAGING_INSTALL_DIR}/lib/cmake/hiprtc") + +################################################################################ +# ROCm SMI +################################################################################ + +ExternalProject_Add( + rocm_smi_lib + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm_smi_lib + SOURCE_DIR "${ROCM_GIT_DIR}/rocm_smi_lib" + DEPENDS + rocm-core-libs + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) + +add_package_path(rocm-smi "${STAGING_INSTALL_DIR}/lib/cmake/rocm-smi") + +################################################################################ +# HIPIFY +################################################################################ + +ExternalProject_Add( + hipify + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/hipify + SOURCE_DIR "${ROCM_GIT_DIR}/HIPIFY" + DEPENDS + amd-llvm + clr + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) + +################################################################################ +# RCCL +################################################################################ + +ExternalProject_Add( + rccl + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rccl + SOURCE_DIR "${ROCM_GIT_DIR}/rccl" + DEPENDS + clr + hipcc + hipify + rocminfo + CMAKE_ARGS + ${DEFAULT_CMAKE_ARGS} + ${FIND_PACKAGE_OPTIONS} + -DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/bin/hipcc + -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} +) ################################################################################ # Testing @@ -189,7 +322,7 @@ install( ) install( - SCRIPT "cmake/custom_install_amdgpu_runtime.cmake" + SCRIPT "cmake/custom_install_amdgpu_runtime.cmake" COMPONENT amdgpu-runtime ) @@ -215,7 +348,7 @@ set(CPACK_PACKAGE_VERSION_MINOR "${ROCM_MINOR_VERSION}") set(CPACK_PACKAGE_VERSION_PATCH "${ROCM_PATCH_VERSION}") set(CPACK_PACKAGE_INSTALL_DIRECTORY "TheRock-amdgpu") set(CPACK_PACKAGE_FILE_NAME "") -set(CPACK_COMPONENTS_ALL +set(CPACK_COMPONENTS_ALL amdgpu-compiler amdgpu-runtime amdgpu-runtime-dev @@ -225,30 +358,30 @@ set(CPACK_COMPONENTS_ALL set(CPACK_ARCHIVE_COMPONENT_INSTALL ON) set(CPACK_ARCHIVE_THREADS 0) set( - CPACK_ARCHIVE_AMDGPU-RUNTIME_FILE_NAME + CPACK_ARCHIVE_AMDGPU-RUNTIME_FILE_NAME "TheRock-amdgpu-runtime-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") set( - CPACK_ARCHIVE_AMDGPU-RUNTIME-DEV_FILE_NAME + CPACK_ARCHIVE_AMDGPU-RUNTIME-DEV_FILE_NAME "TheRock-amdgpu-runtime-dev-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") set( - CPACK_ARCHIVE_AMDGPU-COMPILER_FILE_NAME + CPACK_ARCHIVE_AMDGPU-COMPILER_FILE_NAME "TheRock-amdgpu-compiler-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") include(CPack) cpack_add_component( - amdgpu-runtime + amdgpu-runtime DISPLAY_NAME "AMD GPU Runtime" ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-RUNTIME_FILE_NAME}" ) cpack_add_component( - amdgpu-runtime-dev + amdgpu-runtime-dev DISPLAY_NAME "AMD GPU Development Components" DEPENDS amdgpu-runtime ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-RUNTIME-DEV_FILE_NAME}" ) cpack_add_component( - amdgpu-compiler + amdgpu-compiler DISPLAY_NAME "AMD GPU Compiler" ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-COMPILER_FILE_NAME}" ) diff --git a/README.md b/README.md index 8cf5501..41b6fec 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ pip install CppHeaderParser ``` sudo apt install \ repo git-lfs libnuma-dev ninja-build cmake g++ pkg-config libdrm-dev \ - libelf-dev + libelf-dev xxd libgl1-mesa-dev ``` # Checkout Sources diff --git a/build_tools/apply_patches.sh b/build_tools/apply_patches.sh index 1ff3f2c..e2c11a5 100755 --- a/build_tools/apply_patches.sh +++ b/build_tools/apply_patches.sh @@ -38,3 +38,9 @@ apply_patch ROCT-Thunk-Interface ROCT-Thunk-Interface-link-dl-libs.patch stash_changes ROCR-Runtime apply_patch ROCR-Runtime ROCR-Runtime-intree-build.patch + +stash_changes HIPIFY +apply_patch HIPIFY hipify-install-headers-in-include-hipify.patch + +stash_changes rccl +apply_patch rccl rccl-overwrite-generated-files.patch diff --git a/build_tools/fetch_sources.py b/build_tools/fetch_sources.py index 6f0b477..ca9e2f1 100755 --- a/build_tools/fetch_sources.py +++ b/build_tools/fetch_sources.py @@ -24,21 +24,16 @@ def run(args): [ "repo", "init", - "-u", - "https://github.com/RadeonOpenCompute/ROCm.git", + "--manifest-url", + args.manifest_url, "--depth=1", + "--manifest-branch", + args.manifest_branch ], cwd=repo_dir, ) exec(["repo", "sync", "-j16"] + args.projects, cwd=repo_dir) - # Fixup LLVM. - if "llvm-project" in args.projects: - print("Fixing up llvm-project") - llvm_dir = repo_dir / "llvm-project" - exec(["git", "fetch", "rocm-org", "amd-staging", "--depth=1"], cwd=llvm_dir) - exec(["git", "checkout", "rocm-org/amd-staging"], cwd=llvm_dir) - # Patches. if not args.no_patch: apply_patches(args) @@ -56,7 +51,12 @@ def main(argv): "--dir", type=Path, help="Repo dir", default=DEFAULT_SOURCES_DIR ) parser.add_argument( - "--branch", type=str, help="Branch to sync", default="amd-staging" + "--manifest-url", type=str, help="Manifest repository location of ROCm", + default="https://github.com/nod-ai/ROCm.git" + ) + parser.add_argument( + "--manifest-branch", type=str, help="Branch to sync with repo tool", + default="the-rock-main" ) parser.add_argument("--no-patch", action="store_true", help="Disable patching") parser.add_argument( @@ -66,9 +66,13 @@ def main(argv): default=[ "clr", "HIP", + "HIPIFY", "llvm-project", + "rccl", + "rocm_smi_lib", "rocm-cmake", "rocm-core", + "rocminfo", "ROCR-Runtime", "ROCT-Thunk-Interface", ], diff --git a/cmake/custom_install_amdgpu_compiler.cmake b/cmake/custom_install_amdgpu_compiler.cmake index 20eba4b..aba02a8 100644 --- a/cmake/custom_install_amdgpu_compiler.cmake +++ b/cmake/custom_install_amdgpu_compiler.cmake @@ -16,18 +16,24 @@ set(SO_SUFFIX ".so") file( GLOB_RECURSE LLVM_FILES LIST_DIRECTORIES FALSE - RELATIVE ${STAGING_INSTALL_DIR}/llvm + # hipcc expects clang++ to be under llvm/bin, not just bin. + # but hipcc is in bin. + RELATIVE ${STAGING_INSTALL_DIR} + ${STAGING_INSTALL_DIR}/bin/hipcc* + ${STAGING_INSTALL_DIR}/bin/hipconfig* + ${STAGING_INSTALL_DIR}/bin/rocm_agent_enumerator + ${STAGING_INSTALL_DIR}/bin/rocminfo ${STAGING_INSTALL_DIR}/llvm/amdgcn/* + ${STAGING_INSTALL_DIR}/llvm/bin/*lld* ${STAGING_INSTALL_DIR}/llvm/bin/amdgpu* ${STAGING_INSTALL_DIR}/llvm/bin/clang* - ${STAGING_INSTALL_DIR}/llvm/bin/*lld* ${STAGING_INSTALL_DIR}/llvm/bin/offload-arch* ) -foreach(_relpath ${LLVM_FILES}) +foreach(_relpath ${LLVM_FILES} ${HIPCC_FILES}) cmake_path(GET _relpath PARENT_PATH _parent_rel_path) file( - INSTALL ${STAGING_INSTALL_DIR}/llvm/${_relpath} + INSTALL ${STAGING_INSTALL_DIR}/${_relpath} DESTINATION ${CMAKE_INSTALL_PREFIX}/${_parent_rel_path} USE_SOURCE_PERMISSIONS ) diff --git a/cmake/custom_install_amdgpu_runtime.cmake b/cmake/custom_install_amdgpu_runtime.cmake index 1965c4a..1190572 100644 --- a/cmake/custom_install_amdgpu_runtime.cmake +++ b/cmake/custom_install_amdgpu_runtime.cmake @@ -9,7 +9,6 @@ endif() set(LIB_PREFIX "lib") set(SO_SUFFIX ".so") set(LLVM_STAGING_DIR "${STAGING_INSTALL_DIR}/llvm") -set(RUNTIME_DYNAMIC_STAGING_DIR "${STAGING_INSTALL_DIR}/runtime_dynamic") ################################################################################ # Dynamic runtime files @@ -19,9 +18,9 @@ set(RUNTIME_DYNAMIC_STAGING_DIR "${STAGING_INSTALL_DIR}/runtime_dynamic") file( GLOB_RECURSE LIB_FILES LIST_DIRECTORIES FALSE - RELATIVE ${RUNTIME_DYNAMIC_STAGING_DIR} - ${RUNTIME_DYNAMIC_STAGING_DIR}/lib/*${SO_SUFFIX} - ${RUNTIME_DYNAMIC_STAGING_DIR}/lib/*${SO_SUFFIX}.* + RELATIVE ${STAGING_INSTALL_DIR} + ${STAGING_INSTALL_DIR}/lib/*${SO_SUFFIX} + ${STAGING_INSTALL_DIR}/lib/*${SO_SUFFIX}.* ) list(REMOVE_ITEM LIB_FILES # TODO: Get the hip team to not install old version downloaded files. @@ -31,7 +30,7 @@ list(REMOVE_ITEM LIB_FILES foreach(_relpath ${LIB_FILES}) cmake_path(GET _relpath PARENT_PATH _parent_rel_path) file( - INSTALL ${RUNTIME_DYNAMIC_STAGING_DIR}/${_relpath} + INSTALL ${STAGING_INSTALL_DIR}/${_relpath} DESTINATION ${CMAKE_INSTALL_PREFIX}/${_parent_rel_path} USE_SOURCE_PERMISSIONS ) diff --git a/cmake/custom_install_amdgpu_runtime_dev.cmake b/cmake/custom_install_amdgpu_runtime_dev.cmake index 75658b7..632d499 100644 --- a/cmake/custom_install_amdgpu_runtime_dev.cmake +++ b/cmake/custom_install_amdgpu_runtime_dev.cmake @@ -6,31 +6,28 @@ if(NOT IS_DIRECTORY "${STAGING_INSTALL_DIR}") message(FATAL_ERROR "Did not get STAGING_INSTALL_DIR from super-project") endif() -set(LIB_PREFIX "lib") -set(SO_SUFFIX ".so") - -set(RUNTIME_DYNAMIC_STAGING_DIR "${STAGING_INSTALL_DIR}/runtime_dynamic") - -file(INSTALL ${RUNTIME_DYNAMIC_STAGING_DIR}/include DESTINATION ${CMAKE_INSTALL_PREFIX}) +file(INSTALL ${STAGING_INSTALL_DIR}/include DESTINATION ${CMAKE_INSTALL_PREFIX}) # # Assemble file lists. file( GLOB_RECURSE DEV_FILES LIST_DIRECTORIES FALSE - RELATIVE RUNTIME_DYNAMIC_STAGING_DIR - ${RUNTIME_DYNAMIC_STAGING_DIR}/bin/hipcc_cmake_linker_helper - ${RUNTIME_DYNAMIC_STAGING_DIR}/bin/hipdemangleatp - ${RUNTIME_DYNAMIC_STAGING_DIR}/bin/roc-obj - ${RUNTIME_DYNAMIC_STAGING_DIR}/bin/roc-obj-extract - ${RUNTIME_DYNAMIC_STAGING_DIR}/bin/roc-obj-ls - ${RUNTIME_DYNAMIC_STAGING_DIR}/lib/cmake/* - ${RUNTIME_DYNAMIC_STAGING_DIR}/lib/pkgconfig/* + RELATIVE ${STAGING_INSTALL_DIR} + ${STAGING_INSTALL_DIR}/bin/hipcc_cmake_linker_helper + ${STAGING_INSTALL_DIR}/bin/hipdemangleatp + ${STAGING_INSTALL_DIR}/bin/roc-obj + ${STAGING_INSTALL_DIR}/bin/roc-obj-extract + ${STAGING_INSTALL_DIR}/bin/roc-obj-ls + ${STAGING_INSTALL_DIR}/bin/rocm-smi + ${STAGING_INSTALL_DIR}/lib/cmake/* + ${STAGING_INSTALL_DIR}/lib/pkgconfig/* + ${STAGING_INSTALL_DIR}/libexec/* ) foreach(_relpath ${DEV_FILES}) cmake_path(GET _relpath PARENT_PATH _parent_rel_path) file( - INSTALL ${RUNTIME_DYNAMIC_STAGING_DIR}/${_relpath} + INSTALL ${STAGING_INSTALL_DIR}/${_relpath} DESTINATION ${CMAKE_INSTALL_PREFIX}/${_parent_rel_path} USE_SOURCE_PERMISSIONS ) diff --git a/components/amd-llvm/CMakeLists.txt b/components/amd-llvm/CMakeLists.txt index f06efd9..dacad04 100644 --- a/components/amd-llvm/CMakeLists.txt +++ b/components/amd-llvm/CMakeLists.txt @@ -20,6 +20,22 @@ set(LLVM_EXTERNAL_DEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/devic set(LLVM_EXTERNAL_PROJECTS "amddevice-libs;amdcomgr" CACHE STRING "Enable extra projects" FORCE) set(LLVM_EXTERNAL_AMDCOMGR_SOURCE_DIR "${LLVM_DIR}/amd/comgr") set(LLVM_EXTERNAL_AMDDEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs") +#set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW "llvm/amdgcn-new") +# hipcc expects bit codes under amdgcn. +#set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD "amdgcn") # Now include LLVM's source directory add_subdirectory(${LLVM_DIR}/llvm ${CMAKE_BINARY_DIR}/llvm) + +# add_custom_target( +# amd-bitcodes-symlink ALL +# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +# COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/.. +# COMMAND ${CMAKE_COMMAND} -E create_symlink amdgcn ${CMAKE_INSTALL_PREFIX}/../amdgcn +# ) +install(CODE "execute_process( \ + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/.. \ + COMMAND ${CMAKE_COMMAND} -E create_symlink llvm/amdgcn ${CMAKE_INSTALL_PREFIX}/../amdgcn \ + COMMAND_ERROR_IS_FATAL ANY \ + )" +) diff --git a/components/rocm-core-libs/CMakeLists.txt b/components/rocm-core-libs/CMakeLists.txt index 875db22..56092fa 100644 --- a/components/rocm-core-libs/CMakeLists.txt +++ b/components/rocm-core-libs/CMakeLists.txt @@ -11,9 +11,6 @@ add_subdirectory(${ROCM_GIT_DIR}/rocm-core ${CMAKE_BINARY_DIR}/rocm-core) # Build roct-thunk-interface add_subdirectory(${ROCM_GIT_DIR}/ROCT-Thunk-Interface ${CMAKE_BINARY_DIR}/ROCT-Thunk-Interface) -# Build rocm-cmake -add_subdirectory(${ROCM_GIT_DIR}/rocm-cmake ${CMAKE_BINARY_DIR}/rocm-cmake) - # Build ROCR-Runtime set(BUILD_SHARED_LIBS ON) add_subdirectory(${ROCM_GIT_DIR}/ROCR-Runtime/src ${CMAKE_BINARY_DIR}/ROCR-Runtime) diff --git a/patches/hipify-install-headers-in-include-hipify.patch b/patches/hipify-install-headers-in-include-hipify.patch new file mode 100644 index 0000000..8283df8 --- /dev/null +++ b/patches/hipify-install-headers-in-include-hipify.patch @@ -0,0 +1,39 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 02df74d..ceba0f9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -183,8 +183,8 @@ if (NOT HIPIFY_CLANG_TESTS_ONLY) + if(NOT HIPIFY_INCLUDE_IN_HIP_SDK) + # Install all folders under clang/version/ in CMAKE_INSTALL_PREFIX path. + install( +- DIRECTORY ${LLVM_DIR}/../../clang/${LIB_CLANG_RES}/ +- DESTINATION . ++ DIRECTORY ${LLVM_DIR}/../../clang/${LIB_CLANG_RES}/include/ ++ DESTINATION include/hipify + COMPONENT clang-resource-headers + FILES_MATCHING + PATTERN "*.h" +@@ -198,8 +198,8 @@ if (NOT HIPIFY_CLANG_TESTS_ONLY) + + # install all folders under clang/version/ in CMAKE_INSTALL_PREFIX path + install( +- DIRECTORY ${LLVM_DIR}/../../clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/ +- DESTINATION . ++ DIRECTORY ${LLVM_DIR}/../../clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/include/ ++ DESTINATION include/hipify + COMPONENT clang-resource-headers + FILES_MATCHING + PATTERN "*.h" +diff --git a/packaging/hipify-clang.txt b/packaging/hipify-clang.txt +index 27dae2e..0d75178 100644 +--- a/packaging/hipify-clang.txt ++++ b/packaging/hipify-clang.txt +@@ -9,7 +9,7 @@ install(PROGRAMS @HIPIFY_BIN_INSTALL_PATH@/hipconvertinplace-perl.sh DESTINATION + install(PROGRAMS @HIPIFY_BIN_INSTALL_PATH@/hipconvertinplace.sh DESTINATION @CMAKE_INSTALL_BINDIR@) + install(PROGRAMS @HIPIFY_BIN_INSTALL_PATH@/hipexamine-perl.sh DESTINATION @CMAKE_INSTALL_BINDIR@) + install(PROGRAMS @HIPIFY_BIN_INSTALL_PATH@/hipexamine.sh DESTINATION @CMAKE_INSTALL_BINDIR@) +-install(DIRECTORY @CMAKE_INSTALL_PREFIX@/include/ DESTINATION @CMAKE_INSTALL_INCLUDEDIR@/hipify) ++install(DIRECTORY @CMAKE_INSTALL_PREFIX@/include/hipify/ DESTINATION @CMAKE_INSTALL_INCLUDEDIR@/hipify) + + set (FILE_REORG_BACKWARD_COMPATIBILITY "@FILE_REORG_BACKWARD_COMPATIBILITY@") + if(FILE_REORG_BACKWARD_COMPATIBILITY) diff --git a/patches/rccl-overwrite-generated-files.patch b/patches/rccl-overwrite-generated-files.patch new file mode 100644 index 0000000..1f17674 --- /dev/null +++ b/patches/rccl-overwrite-generated-files.patch @@ -0,0 +1,20 @@ +diff --git a/cmake/Generator.cmake b/cmake/Generator.cmake +index a38cd80..12cf711 100644 +--- a/cmake/Generator.cmake ++++ b/cmake/Generator.cmake +@@ -149,6 +149,7 @@ function(gen_device_table) + ## Generate device table and list all the functions + set(DEVICE_TABLE_H_FILE "${HIPIFY_DIR}/src/collectives/device/device_table.h") + message(STATUS "Generating ${DEVICE_TABLE_H_FILE}") ++ file(REMOVE "${DEVICE_TABLE_H_FILE}") + + ## Declaration of device functions + foreach(func IN LISTS FUNC_LIST) +@@ -218,6 +219,7 @@ function(gen_device_table) + if(COLLTRACE) + set(DEVICE_TABLE_FILE "${HIPIFY_DIR}/src/collectives/device/device_table.cpp") + message(STATUS "Generating ${DEVICE_TABLE_FILE}") ++ file(REMOVE "${DEVICE_TABLE_FILE}") + + file(APPEND ${DEVICE_TABLE_FILE} "#include \"collectives.h\"\n#include \"devcomm.h\"\n\n const char* funcNames[FUNC_INDEX_TOTAL] = {\n") + foreach(func ${FUNC_LIST}) diff --git a/tests/dlopen-hip.c b/tests/dlopen-hip.c index 42b2647..13eba75 100644 --- a/tests/dlopen-hip.c +++ b/tests/dlopen-hip.c @@ -3,7 +3,7 @@ int main(int argc, char **argv) { if (argc != 2) { - printf("Syntax error: Expected library path\n"); + fprintf(stderr, "Syntax error: Expected library path\n"); return 1; } char *lib_path = argv[1]; @@ -11,20 +11,20 @@ int main(int argc, char **argv) { void *h = dlopen(lib_path, RTLD_NOW); int(*hipRuntimeGetVersion)(int*) = (int(*)(int*))dlsym(h, "hipRuntimeGetVersion"); if (!hipRuntimeGetVersion) { - printf("ERROR: Could not resolve symbol hipRuntimeGetVersion\n"); + fprintf(stderr, "ERROR: Could not resolve symbol hipRuntimeGetVersion\n"); return 1; } int version = -1; rc = hipRuntimeGetVersion(&version); if (rc != 0) { - fprintf("ERROR: hipRuntimeGetVersion returned %d\n", rc); + fprintf(stderr, "ERROR: hipRuntimeGetVersion returned %d\n", rc); return 2; } - printf("HIP VERSION: %x\n", version); + fprintf(stderr, "HIP VERSION: %x\n", version); rc = dlclose(h); if (rc != 0) { - printf("ERROR: dlclose(): %d\n", rc); + fprintf(stderr, "ERROR: dlclose(): %d\n", rc); return 3; } return 0;