From cf5e6e91cee713475dd4f7a185326030f8762f5d Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Thu, 30 Jan 2025 16:54:45 +0000 Subject: [PATCH] ci: create a reusable workflow for loading kernel cache --- .github/actions/restore-kernel-cache.yml | 25 ++++++++ .github/workflows/bpf-next-test.yml | 20 +----- .github/workflows/caching-build.yml | 74 ++++------------------ .github/workflows/for-next-test.yml | 78 ++++-------------------- .github/workflows/stable.yml | 20 +----- 5 files changed, 55 insertions(+), 162 deletions(-) create mode 100644 .github/actions/restore-kernel-cache.yml diff --git a/.github/actions/restore-kernel-cache.yml b/.github/actions/restore-kernel-cache.yml new file mode 100644 index 000000000..3a9427fb0 --- /dev/null +++ b/.github/actions/restore-kernel-cache.yml @@ -0,0 +1,25 @@ +name: 'Restore kernel cache' +description: 'Restore a previously built kernel cache, failing if not available' +inputs: + git-repo: + required: true + type: string + branch: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Get hash from repo/branch + run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote ${{ inputs.git-repo }} heads/${{ inputs.branch }} | awk '{print $1}')" >> $GITHUB_ENV + + - name: Load cache + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: | + linux/arch/x86/boot/bzImage + linux/usr/include + linux/**/*.h + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 diff --git a/.github/workflows/bpf-next-test.yml b/.github/workflows/bpf-next-test.yml index f131d2c4f..296a0cc5b 100644 --- a/.github/workflows/bpf-next-test.yml +++ b/.github/workflows/bpf-next-test.yml @@ -95,24 +95,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of sched_ext for-next - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-bpf-1 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git" + branch: for-next # veristat - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml index e8c3da43d..97cbb20a7 100644 --- a/.github/workflows/caching-build.yml +++ b/.github/workflows/caching-build.yml @@ -175,23 +175,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-6.13 | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-6.13 # veristat - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz @@ -264,23 +251,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-6.13 | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-6.13 # The actual build: - run: meson setup build -Dkernel=../linux/arch/x86/boot/bzImage -Dkernel_headers=../linux -Denable_stress=true -Dvng_rw_mount=true -Dextra_sched_args=" ${{ matrix.topo }} ${{ matrix.antistall }}" @@ -338,22 +312,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-6.13 | awk '{print $1}')" >> $GITHUB_ENV - - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-6.13 - run: cargo build --manifest-path rust/${{ matrix.component }}/Cargo.toml - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --network user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml @@ -380,22 +342,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-6.13 | awk '{print $1}')" >> $GITHUB_ENV - # Cache Kernel alone for rust tests - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-6.13 - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --network user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml diff --git a/.github/workflows/for-next-test.yml b/.github/workflows/for-next-test.yml index cb71a72ce..29c2ba5a9 100644 --- a/.github/workflows/for-next-test.yml +++ b/.github/workflows/for-next-test.yml @@ -116,24 +116,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of sched_ext for-next - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-next # veristat - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz @@ -206,24 +192,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of sched_ext for-next - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-next # The actual build: - run: meson setup build -Dkernel=../linux/arch/x86/boot/bzImage -Dkernel_headers=../linux -Denable_stress=true -Dvng_rw_mount=true -Dextra_sched_args=" ${{ matrix.topo }} ${{ matrix.antistall }}" @@ -281,23 +253,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of sched_ext for-next - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-next - run: cargo build --manifest-path rust/${{ matrix.component }}/Cargo.toml - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --network user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml @@ -324,23 +283,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of sched_ext for-next - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - # Cache Kernel alone for rust tests - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-5 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git" + branch: for-next - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --network user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index 554f887a8..e226564f4 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -95,24 +95,10 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # get latest head commit of linux-rolling-stable - - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git heads/linux-rolling-stable | awk '{print $1}')" >> $GITHUB_ENV - - # use cached kernel if available, create after job if not - - name: Cache Kernel - id: cache-kernel - uses: actions/cache@v4 + - uses: ./.github/actions/restore-kernel-cache.yml with: - path: | - linux/arch/x86/boot/bzImage - linux/usr/include - linux/**/*.h - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-stable-1 - - # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} - name: exit if cache stale - run: exit -1 + git-repo: "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" + branch: linux-rolling-stable # veristat - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz