From bf5972b05fa66cceda72083ce82f74e684d15902 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Mon, 5 Aug 2024 22:43:28 -0700 Subject: [PATCH 01/21] Update CK commit hash in requirements.txt for staging --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 96b87492f7..31eb977804 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@15baccf2ecad4fb3498b8acb6bbf58fb5359c7a5 -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@94642acf8ec1b8ce3c0546ddf41b3917bb67d05a -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON google/googletest@v1.14.0 From b93fd3ef65b240fc5b5d2053e5221fd688430685 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Mon, 5 Aug 2024 23:39:15 -0700 Subject: [PATCH 02/21] Update CI base docker to ROCm 6.2 --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89c8065f48..70a025f31e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ RUN dpkg --add-architecture i386 # Install preliminary dependencies RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ - "linux-headers-$(uname -r)" "linux-modules-extra-$(uname -r)" \ apt-utils \ ca-certificates \ curl \ @@ -19,13 +18,13 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn RUN curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg -RUN wget https://repo.radeon.com/amdgpu-install/6.1/ubuntu/jammy/amdgpu-install_6.1.60100-1_all.deb --no-check-certificate +RUN wget wget https://repo.radeon.com/amdgpu-install/6.2/ubuntu/jammy/amdgpu-install_6.2.60200-1_all.deb --no-check-certificate RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ - ./amdgpu-install_6.1.60100-1_all.deb + ./amdgpu-install_6.2.60200-1_all.deb # Add rocm repository -RUN export ROCM_APT_VER=6.1;\ +RUN export ROCM_APT_VER=6.2;\ echo $ROCM_APT_VER &&\ sh -c 'echo deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] https://repo.radeon.com/amdgpu/$ROCM_APT_VER/ubuntu jammy main > /etc/apt/sources.list.d/amdgpu.list' &&\ sh -c 'echo deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] https://repo.radeon.com/rocm/apt/$ROCM_APT_VER jammy main > /etc/apt/sources.list.d/rocm.list' From d689101fcec65df012451e0d31aa18463197329b Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Mon, 5 Aug 2024 23:48:56 -0700 Subject: [PATCH 03/21] Fix typo issues --- Dockerfile | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70a025f31e..bc63e85b9c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn RUN curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg -RUN wget wget https://repo.radeon.com/amdgpu-install/6.2/ubuntu/jammy/amdgpu-install_6.2.60200-1_all.deb --no-check-certificate +RUN wget https://repo.radeon.com/amdgpu-install/6.2/ubuntu/jammy/amdgpu-install_6.2.60200-1_all.deb --no-check-certificate RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ ./amdgpu-install_6.2.60200-1_all.deb diff --git a/Jenkinsfile b/Jenkinsfile index 424c1d8a2f..99af6ed3fc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -194,7 +194,7 @@ def getDockerImage(Map conf=[:]) { env.DOCKER_BUILDKIT=1 def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm - def gpu_arch = "gfx900;gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages + def gpu_arch = "gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages def mlir_build = conf.get("mlir_build", "ON") // always ON def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg USE_MLIR='${mlir_build}' " if(env.CCACHE_HOST) From 57739ff9edb4c69ce1cf1ee8434649ac202ac9b0 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 6 Aug 2024 00:44:12 -0700 Subject: [PATCH 04/21] clean up dockerfile and jenkinsfile --- Dockerfile | 7 +------ Jenkinsfile | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc63e85b9c..2ed4afed04 100755 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,7 @@ ADD dev-requirements.txt /dev-requirements.txt # Install dependencies # TODO: Add --std=c++14 ARG GPU_ARCH=";" -ARG PREFIX=/usr/local +ARG PREFIX=/opt/rocm ARG USE_FIN="OFF" ARG CCACHE_SECONDARY_STORAGE="" ARG CCACHE_DIR="/tmp" @@ -98,11 +98,6 @@ RUN tar zxvf /tmp/ccache.tar.gz -C /tmp/ && mkdir /tmp/ccache-${CCACHE_COMMIT}/b cmake -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. && make -j install && rm -rf /tmp/* RUN ccache -s -# purge existing composable kernel installed with ROCm -# hence cannot use autoremove since it will remove more components -RUN apt-get update && \ -DEBIAN_FRONTEND=noninteractive apt-get purge -y --allow-unauthenticated \ - composablekernel-dev ARG COMPILER_LAUNCHER="" # rbuild is used to trigger build of requirements.txt, dev-requirements.txt RUN if [ "$USE_FIN" = "ON" ]; then \ diff --git a/Jenkinsfile b/Jenkinsfile index 99af6ed3fc..83ac021fa8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -194,9 +194,9 @@ def getDockerImage(Map conf=[:]) { env.DOCKER_BUILDKIT=1 def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm - def gpu_arch = "gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages + def gpu_arch = "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages def mlir_build = conf.get("mlir_build", "ON") // always ON - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg USE_MLIR='${mlir_build}' " + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_TARGETS="${gpu_arch}"" --build-arg USE_MLIR='${mlir_build}' " if(env.CCACHE_HOST) { def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() From e5a0a1351ca6962633772e9b88252a2857cb71dc Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 6 Aug 2024 00:57:05 -0700 Subject: [PATCH 05/21] fix another typo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 83ac021fa8..f5803df3fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -196,7 +196,7 @@ def getDockerImage(Map conf=[:]) def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm def gpu_arch = "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages def mlir_build = conf.get("mlir_build", "ON") // always ON - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_TARGETS="${gpu_arch}"" --build-arg USE_MLIR='${mlir_build}' " + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_TARGETS='${gpu_arch}' --build-arg USE_MLIR='${mlir_build}' " if(env.CCACHE_HOST) { def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() From f974b10e3d5578e3192ee69a834be841600ca823 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 6 Aug 2024 13:51:45 -0700 Subject: [PATCH 06/21] Update CK commit hash to include fix --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 31eb977804..a6c4e2038c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@94642acf8ec1b8ce3c0546ddf41b3917bb67d05a -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@be58e518d8536e96fbfd2200a294ff1aec5d1e2d -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON google/googletest@v1.14.0 From 7a49d8495c2916d6ec05962604303086e6400bad Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 6 Aug 2024 18:47:09 -0700 Subject: [PATCH 07/21] update CK commit hash to include more fixes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6c4e2038c..3a7f94950b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@be58e518d8536e96fbfd2200a294ff1aec5d1e2d -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@b89a5049296184d61b7c825d66c32b019f22819d -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON google/googletest@v1.14.0 From 6a764795759a174b4f44cd762444d344e7950175 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 6 Aug 2024 22:44:07 -0700 Subject: [PATCH 08/21] update dockerfile for dependency installation path --- Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ed4afed04..0e18419079 100755 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,8 @@ ADD dev-requirements.txt /dev-requirements.txt # Install dependencies # TODO: Add --std=c++14 ARG GPU_ARCH=";" -ARG PREFIX=/opt/rocm +# install to /opt/rocm will cause permission issue +ARG PREFIX=/usr/local ARG USE_FIN="OFF" ARG CCACHE_SECONDARY_STORAGE="" ARG CCACHE_DIR="/tmp" @@ -98,6 +99,13 @@ RUN tar zxvf /tmp/ccache.tar.gz -C /tmp/ && mkdir /tmp/ccache-${CCACHE_COMMIT}/b cmake -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. && make -j install && rm -rf /tmp/* RUN ccache -s +# purge existing composable kernel installed with ROCm +# hence cannot use autoremove since it will remove more components +# even purge will remove some other components which is not ideal +RUN apt-get update && \ +DEBIAN_FRONTEND=noninteractive apt-get purge -y --allow-unauthenticated \ + composablekernel-dev + ARG COMPILER_LAUNCHER="" # rbuild is used to trigger build of requirements.txt, dev-requirements.txt RUN if [ "$USE_FIN" = "ON" ]; then \ From 396560647279f07bb3677023e950825e04626bba Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Wed, 7 Aug 2024 10:05:20 -0700 Subject: [PATCH 09/21] Workaround ROCm 6.3 clang-tidy 18 warnings --- .clang-tidy | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.clang-tidy b/.clang-tidy index 8ed34ebb0f..39e6d96b87 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -71,6 +71,19 @@ # -modernize-type-traits # -performance-avoid-endl # +# TODO Code Quality WORKAROUND ROCm 6.2 +# -readability-container-size-empty +# -bugprone-inc-dec-in-conditions +# -readability-avoid-nested-conditional-operator +# -performance-enum-size +# -readability-redundant-member-init +# -bugprone-multi-level-implicit-pointer-conversion +# -clang-analyzer-optin.core.EnumCastOutOfRange +# -cppcoreguidelines-macro-to-enum +# -cppcoreguidelines-owning-memory +# -readability-avoid-return-with-void-value +# -bugprone-unused-local-non-trivial-variable +# Checks: >- *, -abseil-*, @@ -78,9 +91,12 @@ Checks: >- -android-cloexec-fopen, -bugprone-easily-swappable-parameters, -bugprone-exception-escape, + -bugprone-inc-dec-in-conditions, -bugprone-lambda-function-name, -bugprone-macro-parentheses, + -bugprone-multi-level-implicit-pointer-conversion, -bugprone-narrowing-conversions, + -bugprone-unused-local-non-trivial-variable, -bugprone-use-after-move, -cert-env33-c, -cert-err33-c, @@ -90,6 +106,7 @@ Checks: >- -cert-msc51-cpp, -clang-analyzer-alpha.core.CastToStruct, -clang-analyzer-cplusplus.NewDeleteLeaks, + -clang-analyzer-optin.core.EnumCastOutOfRange, -clang-analyzer-optin.performance.Padding, -clang-diagnostic-extern-c-compat, -clang-diagnostic-unused-command-line-argument, @@ -100,11 +117,13 @@ Checks: >- -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-explicit-virtual-functions, -cppcoreguidelines-init-variables, + -cppcoreguidelines-macro-to-enum, -cppcoreguidelines-macro-usage, -cppcoreguidelines-misleading-capture-default-by-value, -cppcoreguidelines-missing-std-forward, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-owning-memory, -cppcoreguidelines-prefer-member-initializer, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, -cppcoreguidelines-pro-bounds-constant-array-index, @@ -168,11 +187,15 @@ Checks: >- -modernize-concat-nested-namespaces, -modernize-unary-static-assert, -performance-avoid-endl, + -performance-enum-size, + -readability-avoid-nested-conditional-operator, -performance-no-automatic-move, -performance-unnecessary-copy-initialization, -performance-unnecessary-value-param, + -readability-avoid-return-with-void-value, -readability-avoid-unconditional-preprocessor-if, -readability-container-data-pointer, + -readability-container-size-empty, -readability-convert-member-functions-to-static, -readability-else-after-return, -readability-function-cognitive-complexity, @@ -182,6 +205,7 @@ Checks: >- -readability-named-parameter, -readability-qualified-auto, -readability-redundant-declaration, + -readability-redundant-member-init, -readability-redundant-string-init, -readability-simplify-boolean-expr, -readability-suspicious-call-argument, From 26ab5dbce020fa78ba73b0dfe6141f7af5359c94 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Wed, 7 Aug 2024 22:43:13 -0700 Subject: [PATCH 10/21] Need to install before running tests to override existing libs come with rocm installation --- CMakeLists.txt | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98201516ad..21f96a0d77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,7 +127,7 @@ if(MIOPEN_STRIP_SYMBOLS AND NOT WIN32 AND NOT APPLE) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s") endif() -rocm_setup_version(VERSION 3.2.0) +rocm_setup_version(VERSION 3.3.0) list( APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) include(TargetFlags) diff --git a/Jenkinsfile b/Jenkinsfile index f5803df3fa..1bacc4c210 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -545,7 +545,7 @@ pipeline { Bf16_flags = " -DMIOPEN_TEST_BFLOAT16=On" Int8_flags = " -DMIOPEN_TEST_INT8=On" Full_test = " -DMIOPEN_TEST_ALL=On" - Smoke_targets = " check MIOpenDriver" + Smoke_targets = " install check MIOpenDriver" NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" } From c4bf60945469889c339f861b0564ab1135cdf80c Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 00:03:31 -0700 Subject: [PATCH 11/21] Update Jenkins file --- Jenkinsfile | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1bacc4c210..5235beea9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -545,7 +545,7 @@ pipeline { Bf16_flags = " -DMIOPEN_TEST_BFLOAT16=On" Int8_flags = " -DMIOPEN_TEST_INT8=On" Full_test = " -DMIOPEN_TEST_ALL=On" - Smoke_targets = " install check MIOpenDriver" + Smoke_targets = " check MIOpenDriver" NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" } @@ -648,7 +648,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets) + buildHipClangJobAndReboot(make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx90a') { @@ -661,7 +661,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx908') { @@ -674,7 +674,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx94X') { @@ -687,7 +687,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } } @@ -711,7 +711,7 @@ pipeline { NOCOMGR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Debug NOMLIR gfx90a') { @@ -728,7 +728,7 @@ pipeline { NOMLIR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Debug NOCK gfx90a Build-Only') { @@ -741,7 +741,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "") + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "", build_install: "true") } } stage('Fp32 Hip Debug Embedded Vega20') { @@ -757,7 +757,7 @@ pipeline { Embedded_flags = "-DMIOPEN_EMBED_DB='gfx906_60'" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Static gfx90a') { @@ -770,7 +770,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF') + buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF', build_install: "true") } } stage('Fp32 Hip Normal-Find gfx90a') { @@ -787,7 +787,7 @@ pipeline { execute_cmd = "bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal") + buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal", build_install: "true") } } stage('Fp32 Hip Fast-Find gfx90a') { @@ -804,7 +804,7 @@ pipeline { execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd) + buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd, build_install: "true") } } stage('Fp32 Hip gfx90a') { @@ -830,7 +830,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On") + buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On", build_install: "true") } } } @@ -850,7 +850,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip Vega20') { @@ -863,7 +863,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx908') { @@ -876,7 +876,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip gfx908') { @@ -889,7 +889,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx90a') { @@ -902,7 +902,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip gfx90a') { @@ -915,7 +915,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx94X') { @@ -928,7 +928,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } stage('Bf16 Hip gfx94X') { @@ -941,7 +941,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } } From a75a6c1e4f46cdf96db2c75ad3b2b89e96cb44e0 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 01:12:13 -0700 Subject: [PATCH 12/21] Update dockerfile --- Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e18419079..5670e9167f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,12 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ gnupg2 \ wget +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ + "linux-headers-$(uname -r)" "linux-modules-extra-$(uname -r)" + +RUN usermod -a -G render,video $LOGNAME + #Add gpg keys ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn RUN curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg @@ -104,7 +110,8 @@ RUN ccache -s # even purge will remove some other components which is not ideal RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get purge -y --allow-unauthenticated \ - composablekernel-dev + composablekernel-dev \ + miopen-hip ARG COMPILER_LAUNCHER="" # rbuild is used to trigger build of requirements.txt, dev-requirements.txt @@ -122,4 +129,4 @@ RUN pip3 install -r /doc-requirements.txt # Composable Kernel requires this version cmake RUN pip3 install --upgrade cmake==3.27.5 -RUN groupadd -f render +RUN groupadd -f render \ No newline at end of file From 654489fe4aaf501b76e7a95f8eb3aaf4334322c4 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 01:20:09 -0700 Subject: [PATCH 13/21] Workaround hipRTC warning issue in #3188 --- src/comgr.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/comgr.cpp b/src/comgr.cpp index aa53b71bb5..b508bced2f 100644 --- a/src/comgr.cpp +++ b/src/comgr.cpp @@ -701,6 +701,7 @@ void BuildAsm(const std::string& name, #define WORKAROUND_ISSUE_HIPRTC_HIPRTC_HEADER_H 1 // See SWDEV-307838, issue #1648. #define WORKAROUND_ISSUE_1674 (HIP_PACKAGE_VERSION_FLAT >= 5003022305ULL) +#define WORKAROUND_ISSUE_3188 (HIP_PACKAGE_VERSION_FLAT >= 6002041133ULL) // See WORKAROUND_SWDEV_413293 in ./CmakeLists.txt #define WORKAROUND_SWDEV_413293 MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK @@ -975,6 +976,9 @@ void BuildHip(const std::string& name, #endif #if WORKAROUND_ISSUE_1674 opts.push_back("-Wno-gnu-line-marker"); +#endif +#if WORKAROUND_ISSUE_3188 + opts.push_back("-Wno-pass-failed"); #endif opts.push_back("-Wno-cuda-compat"); opts.push_back("-fno-gpu-rdc"); From 6b774f4800d916a0dc29cb555c0305711c39f5ff Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 01:21:49 -0700 Subject: [PATCH 14/21] fix dockerfile issue --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5670e9167f..ab1cb5b056 100755 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ "linux-headers-$(uname -r)" "linux-modules-extra-$(uname -r)" -RUN usermod -a -G render,video $LOGNAME - #Add gpg keys ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn RUN curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg @@ -129,4 +127,4 @@ RUN pip3 install -r /doc-requirements.txt # Composable Kernel requires this version cmake RUN pip3 install --upgrade cmake==3.27.5 -RUN groupadd -f render \ No newline at end of file +RUN groupadd -f render,video \ No newline at end of file From d21ab31be69d168af931a43bb29c2c6cae9f9b96 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 10:12:22 -0700 Subject: [PATCH 15/21] Update Dockerfile --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ab1cb5b056..fb272596aa 100755 --- a/Dockerfile +++ b/Dockerfile @@ -127,4 +127,7 @@ RUN pip3 install -r /doc-requirements.txt # Composable Kernel requires this version cmake RUN pip3 install --upgrade cmake==3.27.5 -RUN groupadd -f render,video \ No newline at end of file +# groupadd can add one group a time +RUN groupadd -f render +RUN groupadd -f video +RUN usermod -a -G render,video root \ No newline at end of file From 28e493df63d04425dcf5e0eab380dee0e9c6cb0e Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 8 Aug 2024 23:01:06 -0700 Subject: [PATCH 16/21] Update CK to include revert of API change --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a7f94950b..f7787cbaa3 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@b89a5049296184d61b7c825d66c32b019f22819d -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@f2ba10b826d5e73dc3dddd83abe7240395399837 -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON google/googletest@v1.14.0 From 6358692f2fcd6c72a168ba908d3f486f64caefc8 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Mon, 12 Aug 2024 17:36:01 -0700 Subject: [PATCH 17/21] issue-3192: add -fsanitize=undefined back when miirCreateHandle issue is resolved --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5235beea9d..b9014b2048 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,7 +30,8 @@ def cmake_build(Map conf=[:]){ def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") def make_targets = conf.get("make_targets","check") - def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") + // TODO: issue-3192: add -fsanitize=undefined back when miirCreateHandle issue is resolved + def debug_flags = "-g -fno-omit-frame-pointer -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") def prefixpath = conf.get("prefixpath","/opt/rocm") def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") From f43075fd86a15cbac8095b7519ba179f1f9e7c37 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Tue, 13 Aug 2024 07:35:10 -0700 Subject: [PATCH 18/21] relax on timeout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b9014b2048..29dd34bee5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -285,7 +285,7 @@ def buildHipClangJob(Map conf=[:]){ } withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 210, unit:'MINUTES') + timeout(time: 300, unit:'MINUTES') { if (lfs_pull) { sh "git lfs pull --exclude=" From 605c4a696fcaf56921598d1331001aaeb12614a5 Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Thu, 15 Aug 2024 14:46:00 +0000 Subject: [PATCH 19/21] Disable MLIR when building debug to workaround sanitizer issue --- Jenkinsfile | 13 ++++++++++--- test/nightlies/JenkinsfileNightlyAux | 3 ++- test/nightlies/JenkinsfileNightlyConv2D | 3 ++- test/nightlies/JenkinsfileNightlyConv2Daux | 3 ++- test/nightlies/JenkinsfileNightlyFastFullConv2D | 3 ++- test/nightlies/JenkinsfileNightlyFusions | 3 ++- test/nightlies/JenkinsfileNightlyRNN | 3 ++- test/nightlies/JenkinsfileNightlyReduce | 3 ++- test/nightlies/JenkinsfileNightlyTensorOps | 3 ++- 9 files changed, 26 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 424c1d8a2f..a54735e8c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,14 +33,21 @@ def cmake_build(Map conf=[:]){ def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") def prefixpath = conf.get("prefixpath","/opt/rocm") + def build_type_debug = (conf.get("build_type",'release') == 'debug') + def code_conv_enabled = conf.get("codecov", false) + def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + if (build_type_debug || code_conv_enabled) + { + mlir_args = " -DMIOPEN_USE_MLIR=OFF" + } + def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","") def build_fin = conf.get("build_fin", "OFF") setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} " - def build_type_debug = (conf.get("build_type",'release') == 'debug') - //cmake_env can overwrite default CXX variables. def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","") @@ -72,7 +79,7 @@ def cmake_build(Map conf=[:]){ test_flags = " --disable-verification-cache " + test_flags } - if(conf.get("codecov", false)){ //Need + if(code_conv_enabled){ //Need setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args }else if(build_type_debug){ setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args diff --git a/test/nightlies/JenkinsfileNightlyAux b/test/nightlies/JenkinsfileNightlyAux index d1b7e70951..feb6ab5dc7 100644 --- a/test/nightlies/JenkinsfileNightlyAux +++ b/test/nightlies/JenkinsfileNightlyAux @@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 8, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyConv2D b/test/nightlies/JenkinsfileNightlyConv2D index 5b67507376..809cfff95c 100644 --- a/test/nightlies/JenkinsfileNightlyConv2D +++ b/test/nightlies/JenkinsfileNightlyConv2D @@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 8, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyConv2Daux b/test/nightlies/JenkinsfileNightlyConv2Daux index e6439313f7..f967352b21 100644 --- a/test/nightlies/JenkinsfileNightlyConv2Daux +++ b/test/nightlies/JenkinsfileNightlyConv2Daux @@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 8, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyFastFullConv2D b/test/nightlies/JenkinsfileNightlyFastFullConv2D index eaa703b3b1..a94992002b 100644 --- a/test/nightlies/JenkinsfileNightlyFastFullConv2D +++ b/test/nightlies/JenkinsfileNightlyFastFullConv2D @@ -43,7 +43,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 12, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_ENABLE_LOGGING_CMD=1 MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyFusions b/test/nightlies/JenkinsfileNightlyFusions index 84d8329c6a..269480baf7 100644 --- a/test/nightlies/JenkinsfileNightlyFusions +++ b/test/nightlies/JenkinsfileNightlyFusions @@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 8, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyRNN b/test/nightlies/JenkinsfileNightlyRNN index b299f832cc..33ac860053 100644 --- a/test/nightlies/JenkinsfileNightlyRNN +++ b/test/nightlies/JenkinsfileNightlyRNN @@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 8, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyReduce b/test/nightlies/JenkinsfileNightlyReduce index a7d6c6becb..c2b14e8f79 100644 --- a/test/nightlies/JenkinsfileNightlyReduce +++ b/test/nightlies/JenkinsfileNightlyReduce @@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 6, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}" diff --git a/test/nightlies/JenkinsfileNightlyTensorOps b/test/nightlies/JenkinsfileNightlyTensorOps index e19a0a9f18..d326c64f9f 100644 --- a/test/nightlies/JenkinsfileNightlyTensorOps +++ b/test/nightlies/JenkinsfileNightlyTensorOps @@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){ timeout(time: 6, unit: 'HOURS') { sh "echo \$HSA_ENABLE_SDMA" - sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .." sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}" sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}" From 16dba899e68255824f0747b10680f8fe976fe476 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 15 Aug 2024 10:38:49 -0700 Subject: [PATCH 20/21] Update CK commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f7787cbaa3..d9c578224f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@f2ba10b826d5e73dc3dddd83abe7240395399837 -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@e2eb0418913b94069e3320b5ef4b8935620f556a -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON google/googletest@v1.14.0 From cd481f75011efa3a0ba75d88a914036e90d3c6ae Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Mon, 19 Aug 2024 14:13:43 -0700 Subject: [PATCH 21/21] Revert "issue-3192: add -fsanitize=undefined back when miirCreateHandle issue is resolved" This reverts commit 6358692f2fcd6c72a168ba908d3f486f64caefc8. --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c96c269bbc..6d85a6f7d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,8 +30,7 @@ def cmake_build(Map conf=[:]){ def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") def make_targets = conf.get("make_targets","check") - // TODO: issue-3192: add -fsanitize=undefined back when miirCreateHandle issue is resolved - def debug_flags = "-g -fno-omit-frame-pointer -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") + def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") def prefixpath = conf.get("prefixpath","/opt/rocm") def build_type_debug = (conf.get("build_type",'release') == 'debug')