diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index dd2d549..b53608d 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -4,20 +4,21 @@ on: workflow_dispatch: inputs: package_version: - required: true type: string default: ADHOCBUILD workflow_call: inputs: package_version: - required: true type: string + default: ADHOCBUILD jobs: build_linux_packages: name: Build Linux Packages - runs-on: nod-ai-shared-cpubuilder-manylinux-x86_64 + runs-on: azure-cpubuilder-linux-scale + container: + image: ghcr.io/nod-ai/therock_build_manylinux_x86_64:main strategy: fail-fast: true env: @@ -35,21 +36,21 @@ jobs: ccache -z mkdir -p $CCACHE_DIR cmake --version + echo "Installed Python versions:" + ls -d /opt/python + echo "python: $(which python), python3: $(which python3)" + echo "Git version: $(git --version)" # TODO: We shouldn't be using a cache on actual release branches, but it # really helps for iteration time. - name: Enable cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: ${{ env.CACHE_DIR }} key: linux-build-packages-manylinux-v2-${{ github.sha }} restore-keys: | linux-build-packages-manylinux-v2- - - name: Install Deps - run: | - ./build_tools/ci_install_build_deps.sh - - name: Fetch sources run: | ./build_tools/fetch_sources.py --depth 1 @@ -103,7 +104,7 @@ jobs: if-no-files-found: warn - name: Save cache - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 if: always() with: path: ${{ env.CACHE_DIR }} diff --git a/.github/workflows/build_python_packages.yml b/.github/workflows/build_python_packages.yml index 7bfdfc3..df81370 100644 --- a/.github/workflows/build_python_packages.yml +++ b/.github/workflows/build_python_packages.yml @@ -22,7 +22,9 @@ on: jobs: build_python_packages: name: Build Python Packages - runs-on: nod-ai-shared-cpubuilder-manylinux-x86_64 + runs-on: azure-cpubuilder-linux-scale + container: + image: ghcr.io/nod-ai/therock_build_manylinux_x86_64:main strategy: fail-fast: true env: @@ -40,21 +42,21 @@ jobs: ccache -z mkdir -p $CCACHE_DIR cmake --version + echo "Installed Python versions:" + ls -d /opt/python + echo "python: $(which python), python3: $(which python3)" + echo "Git version: $(git --version)" # TODO: We shouldn't be using a cache on actual release branches, but it # really helps for iteration time. - name: Enable cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: ${{ env.CACHE_DIR }} key: build-python-manylinux-v2-${{ github.sha }} restore-keys: | build-python-manylinux-v2- - - name: Install Deps - run: | - ./build_tools/ci_install_build_deps.sh - - name: Fetch sources run: | ./build_tools/fetch_sources.py --depth 1 @@ -95,7 +97,7 @@ jobs: if-no-files-found: warn - name: Save cache - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 if: always() with: path: ${{ env.CACHE_DIR }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35e71ca..f4e91b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,8 @@ concurrency: jobs: build_linux_packages: name: Build Linux Packages - needs: setup uses: ./.github/workflows/build_linux_packages.yml build_python_packages: name: Build Python Packages - needs: setup uses: ./.github/workflows/build_python_packages.yml diff --git a/build_tools/ci_install_build_deps.sh b/build_tools/ci_install_build_deps.sh index c7580f8..d192d39 100755 --- a/build_tools/ci_install_build_deps.sh +++ b/build_tools/ci_install_build_deps.sh @@ -2,5 +2,5 @@ curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo chmod a+x /usr/local/bin/repo -yum install -y numactl-devel elfutils-libelf-devel vim-common git-lfs +yum install -y numactl-devel elfutils-libelf-devel vim-common git-lfs ninja-build pip install CppHeaderParser diff --git a/build_tools/fetch_sources.py b/build_tools/fetch_sources.py index f5dcc8e..9080add 100755 --- a/build_tools/fetch_sources.py +++ b/build_tools/fetch_sources.py @@ -41,6 +41,8 @@ def run(args): ) exec(["repo", "sync", "-j16"] + args.projects, cwd=repo_dir) + populate_ancillary_sources(args) + # Patches. if not args.no_patch: apply_patches(args) @@ -52,6 +54,22 @@ def apply_patches(args): exec([script], cwd=args.dir) +def populate_ancillary_sources(args): + """Various subprojects have their own mechanisms for populating ancillary sources + needed to build. There is often something in CMake that attempts to automate it, + but it is also often broken. So we just do the right thing here as a transitionary + step to fixing the underlying software packages.""" + populate_submodules_if_exists(args.dir / "rocprofiler-register") + + +def populate_submodules_if_exists(git_dir: Path): + if not git_dir.exists(): + print(f"Not populating submodules for {git_dir} (does not exist)") + return + print(f"Populating submodules for {git_dir}:") + exec(["git", "submodule", "update", "--init"], cwd=git_dir) + + def main(argv): parser = argparse.ArgumentParser(prog="fetch_sources") parser.add_argument( @@ -61,7 +79,7 @@ def main(argv): "--manifest-url", type=str, help="Manifest repository location of ROCm", - default="http://github.com/ROCm/ROCm.git", + default="https://github.com/ROCm/ROCm.git", ) parser.add_argument( "--manifest-name",