From 65a8b0901727957a2ed41d84133303a8b65038ea Mon Sep 17 00:00:00 2001 From: Berne Campbell Date: Mon, 5 Feb 2024 21:58:08 +1100 Subject: [PATCH] Refactor into reusable workflows - Normally will just build kubler images - For now manually kick off early statges, or chains --- .github/workflows/builders.yml | 267 +++++++++++ .github/workflows/chain-builders.yml | 9 + .github/workflows/chain-full.yml | 13 + .github/workflows/chain-portage.yml | 11 + .../workflows/{main.yml => kubler-build.yml} | 451 +----------------- .github/workflows/portage.yml | 114 +++++ .github/workflows/setup.yml | 100 ++++ 7 files changed, 515 insertions(+), 450 deletions(-) create mode 100644 .github/workflows/builders.yml create mode 100644 .github/workflows/chain-builders.yml create mode 100644 .github/workflows/chain-full.yml create mode 100644 .github/workflows/chain-portage.yml rename .github/workflows/{main.yml => kubler-build.yml} (57%) create mode 100644 .github/workflows/portage.yml create mode 100644 .github/workflows/setup.yml diff --git a/.github/workflows/builders.yml b/.github/workflows/builders.yml new file mode 100644 index 0000000..8c2e826 --- /dev/null +++ b/.github/workflows/builders.yml @@ -0,0 +1,267 @@ +name: Bake Builders +on: + #push: + # branches: + # - main + workflow_dispatch: + workflow_call: + +# Jobs run in parallel +# Jobs are independent with separate file systems, IP addresses, etc. +jobs: + stage3: + runs-on: ubuntu-latest + needs: + - setup + - kubler-portage + env: + KUBLER_IMAGE: ghcr.io/${{ github.repository }} + PORTAGE_DATE: ${{ needs.setup.outputs.portage_date }} + PORTAGE_IMAGE: ${{ needs.kubler-portage.outputs.portage_image }} + strategy: + fail-fast: false + matrix: + include: + - name: bob + stage3: + base: gentoo/stage3:hardened-nomultilib + tmp_image: ghcr.io/${{ github.repository }}/tmp/stage3-amd64-hardened-nomultilib-openrc:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/stage3-amd64-hardened-nomultilib-openrc:${{ needs.setup.outputs.portage_date }} + cache: stage3-amd64-hardened-nomultilib-openrc + goss: stage3/amd64-hardened-nomultilib-openrc + core: + def_chost: x86_64-pc-linux-gnu + tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-core:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/bob-core:${{ needs.setup.outputs.portage_date }} + cache: bob-core + goss: core/bob-core + bob: + target: bob + tmp_image: ghcr.io/${{ github.repository }}/tmp/bob:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/bob:${{ needs.setup.outputs.portage_date }} + cache: bob + goss: builder/bob + - name: bob-musl + stage3: + base: gentoo/stage3:musl-hardened + tmp_image: ghcr.io/${{ github.repository }}/tmp/stage3-amd64-musl-hardened:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/stage3-amd64-musl-hardened:${{ needs.setup.outputs.portage_date }} + cache: stage3-amd64-musl-hardened + goss: stage3/amd64-musl-hardened + core: + def_chost: x86_64-gentoo-linux-musl + tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-musl-core:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/bob-musl-core:${{ needs.setup.outputs.portage_date }} + cache: bob-musl-core + goss: core/bob-musl-core + bob: + target: bob-musl + tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-musl:${{ needs.setup.outputs.portage_date }} + image: ghcr.io/${{ github.repository }}/bob-musl:${{ needs.setup.outputs.portage_date }} + cache: bob-musl + goss: builder/bob-musl + steps: + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:latest + platforms: all + + - name: 🐋 Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + # This breaks kubler https://github.com/edannenberg/kubler/issues/215 + # Sets up `docker build` command as an alias to `docker buildx` (default `false`) + install: true + + - name: 🐋 Docker Login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check out repository code + uses: actions/checkout@v4 + + - uses: oras-project/setup-oras@v1 + + - name: 🐋 Docker Pull stage3 + id: stage3-pull + continue-on-error: true + run: | + set -eu + docker pull "${{ matrix.stage3.image }}" + + - name: 🐋🍳 Docker Bake Kubler stage3 - test pulled + id: stage3-pull-test + if: ${{ steps.stage3-pull.outcome == 'success' }} + continue-on-error: true + run: | + set -eux + docker run --rm "${{ matrix.stage3.image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.stage3.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.stage3.image }}" eselect profile show + cd "${{ matrix.stage3.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.stage3.image }}" validate --color + + - name: 🐋🍳 Docker Bake Kubler stage3 - list targets + if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} + uses: docker/bake-action/subaction/list-targets@v4 + with: + workdir: bob-stage3 + + - name: 🐋🍳 Docker Bake Kubler stage3 + if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} + uses: docker/bake-action@v4 + with: + workdir: bob-stage3 + push: true + # In the `docker-bake.hcl` + # There is a variable `BASE_TAG` and `BASE_IMAGE`. `BASE_IMAGE` interpolates `BASE_TAG`i + # These can be set with env vars when using `docker buildx bake` command + # For the gentoo-stage3 target there's no `BASE_TAG` arg, only `BASE_IMAGE` which defaults to `BASE_IMAGE` var + # When using bake-action we need to set the `BASE_IMAGE` arg, setting `BASE_TAG` arg won't do what we want. + set: | + gentoo-stage3.args.BASE_IMAGE=${{ matrix.stage3.base }} + gentoo-stage3.args.PORTAGE=${{ env.PORTAGE_IMAGE }} + gentoo-stage3.tags=${{ matrix.stage3.tmp_image }} + gentoo-stage3.cache-from=type=gha,scope=portage + gentoo-stage3.cache-from=type=gha,scope=${{ matrix.stage3.cache }} + gentoo-stage3.cache-to=type=gha,scope=${{ matrix.stage3.cache }} + + - name: 🐋🍳 Docker Bake Kubler stage3 - test baked + if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} + run: | + set -eux + docker run --rm "${{ matrix.stage3.tmp_image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.stage3.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.stage3.tmp_image }}" eselect profile show + cd "${{ matrix.stage3.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.stage3.tmp_image }}" validate --color + + - name: 🐋🍳 Docker Bake Kubler stage3 - oras cp + if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} + run: | + set -eux + oras cp -v "${{ matrix.stage3.tmp_image }}" "${{ matrix.stage3.image }}" + + - name: 🐋 Docker Pull bob-core + id: bob-core-pull + continue-on-error: true + run: | + set -eu + docker pull "${{ matrix.core.image }}" + + - name: 🐋🍳 Docker Bake Kubler bob-core - test pulled + id: bob-core-pull-test + if: ${{ steps.bob-core-pull.outcome == 'success' }} + run: | + set -eux + docker run --rm "${{ matrix.core.image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.core.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.core.image }}" eselect profile show + docker run --rm "${{ matrix.core.image }}" ls -l /var/cache + cd "${{ matrix.core.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.core.image }}" validate --color + + - name: 🐋🍳 Docker Bake Kubler bob-core - list targets + if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} + uses: docker/bake-action/subaction/list-targets@v4 + with: + workdir: bob-core + + - name: 🐋🍳 Docker Bake Kubler bob-core + if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} + uses: docker/bake-action@v4 + with: + workdir: bob-core + push: true + set: | + core.args.DEF_CHOST=${{ matrix.core.def_chost }} + core.args.BASE_IMAGE=${{ matrix.stage3.image }} + core.tags=${{ matrix.core.tmp_image }} + core.cache-from=type=gha,scope=portage + core.cache-from=type=gha,scope=${{ matrix.stage3.cache }} + core.cache-from=type=gha,scope=${{ matrix.core.cache }} + core.cache-to=type=gha,scope=${{ matrix.core.cache }} + + - name: 🐋🍳 Docker Bake Kubler bob-core - test baked + if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} + run: | + set -eux + docker run --rm "${{ matrix.core.tmp_image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.core.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.core.tmp_image }}" eselect profile show + docker run --rm "${{ matrix.core.tmp_image }}" ls -l /var/cache + cd "${{ matrix.core.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.core.tmp_image }}" validate --color + + - name: 🐋🍳 Docker Bake Kubler bob-core - oras cp + if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} + run: | + set -eux + oras cp -v "${{ matrix.core.tmp_image }}" "${{ matrix.core.image }}" + + - name: 🐋 Docker Pull bob + id: bob-pull + continue-on-error: true + run: | + set -eu + docker pull "${{ matrix.bob.image }}" + + - name: 🐋 Docker Build kubler builder - bob - test pulled + id: bob-pull-test + if: ${{ steps.bob-pull.outcome == 'success' }} + run: | + set -eux + docker run --rm "${{ matrix.bob.image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.bob.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.bob.image }}" eselect profile show + docker run --rm "${{ matrix.bob.image }}" ls -l /var/cache + docker run --rm "${{ matrix.bob.image }}" ls -l /var/cache/eix + docker run --rm "${{ matrix.bob.image }}" eix --selected -c + cd "${{ matrix.bob.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.bob.image }}" validate --color + + - name: 🐋 Docker Build kubler builder - bob - list targets + if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} + uses: docker/bake-action/subaction/list-targets@v4 + with: + workdir: ${{ matrix.bob.goss }} + + - name: 🐋🍳 Docker Bake Kubler builder - bob + if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} + uses: docker/bake-action@v4 + with: + workdir: ${{ matrix.bob.goss }} + push: true + set: | + ${{matrix.bob.target}}.args.BASE_IMAGE=${{ matrix.core.image }} + ${{matrix.bob.target}}.tags=${{ matrix.bob.tmp_image }} + ${{matrix.bob.target}}.cache-from=type=gha,scope=portage + ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.stage3.cache }} + ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.core.cache }} + ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.bob.cache }} + ${{matrix.bob.target}}.cache-to=type=gha,scope=${{ matrix.bob.cache }} + + - name: 🐋 Docker Build kubler builder - bob - test baked + if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} + run: | + set -eux + docker run --rm "${{ matrix.bob.tmp_image }}" cat /etc/gentoo-release + docker run --rm "${{ matrix.bob.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" + docker run --rm "${{ matrix.bob.tmp_image }}" eselect profile show + docker run --rm "${{ matrix.bob.tmp_image }}" ls -l /var/cache + docker run --rm "${{ matrix.bob.tmp_image }}" ls -l /var/cache/eix + docker run --rm "${{ matrix.bob.tmp_image }}" eix --selected -c + cd "${{ matrix.bob.goss }}" || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.bob.tmp_image }}" validate --color + + - name: 🐋🍳 Docker Bake Kubler builder - bob - oras cp + if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} + run: | + set -eux + oras cp -v "${{ matrix.bob.tmp_image }}" "${{ matrix.bob.image }}" diff --git a/.github/workflows/chain-builders.yml b/.github/workflows/chain-builders.yml new file mode 100644 index 0000000..1405508 --- /dev/null +++ b/.github/workflows/chain-builders.yml @@ -0,0 +1,9 @@ +name: Chain Builders-Kubler +on: + workflow_dispatch: + +jobs: + call-builders: + uses: ./.github/builders.yml + call-kubler-build: + uses: ./.github/kubler-build.yml diff --git a/.github/workflows/chain-full.yml b/.github/workflows/chain-full.yml new file mode 100644 index 0000000..23fce85 --- /dev/null +++ b/.github/workflows/chain-full.yml @@ -0,0 +1,13 @@ +name: Full Chain +on: + workflow_dispatch: + +jobs: + call-setup: + uses: ./.github/setup.yml + call-portage: + uses: ./.github/portage.yml + call-builders: + uses: ./.github/builders.yml + call-kubler-build: + uses: ./.github/kubler-build.yml diff --git a/.github/workflows/chain-portage.yml b/.github/workflows/chain-portage.yml new file mode 100644 index 0000000..fe975ea --- /dev/null +++ b/.github/workflows/chain-portage.yml @@ -0,0 +1,11 @@ +name: Chain Portage-Builders-Kubler +on: + workflow_dispatch: + +jobs: + call-portage: + uses: ./.github/portage.yml + call-builders: + uses: ./.github/builders.yml + call-kubler-build: + uses: ./.github/kubler-build.yml diff --git a/.github/workflows/main.yml b/.github/workflows/kubler-build.yml similarity index 57% rename from .github/workflows/main.yml rename to .github/workflows/kubler-build.yml index 20b32af..1e30adc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/kubler-build.yml @@ -4,460 +4,11 @@ on: branches: - main workflow_dispatch: + workflow_call: # Jobs run in parallel # Jobs are independent with separate file systems, IP addresses, etc. jobs: - setup: - name: Setup Kubler - runs-on: ubuntu-latest - outputs: - portage_date: ${{ steps.portage_date.outputs.portage_date }} - bob_stage3_date: ${{ steps.stage3.outputs.bob_stage3_date }} }} - bob_musl_stage3_date: ${{ steps.stage3.outputs.bob_musl_stage3_date }} }} - env: - KUBLER_IMAGE: ghcr.io/${{ github.repository }} - steps: - - name: inspect - run: | - set -eux - echo "${{ github.workspace }}" - ls -la "${{ github.workspace }}" - type systemctl - which systemctl - file $(which systemctl) - ldd $(which systemctl) - ls -la /var/run/dbus/system_bus_socket || true - pidof systemd || true - ps -ef - ps -ef --forest || true - id - uname -a - pwd - echo $HOME - env | grep USER - cd - pwd - mount - - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:latest - platforms: all - - - name: 🐋 Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - with: - # This breaks kubler https://github.com/edannenberg/kubler/issues/215 - # Sets up `docker build` command as an alias to `docker buildx` (default `false`) - install: true - - - name: 🐋 Docker Login - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - # this is the default, end of job will logout - #logout: true - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: 🐋🍳 Docker Bake Kubler - #if: ${{ steps.portage-pull.outcome == 'failure' }} - uses: docker/bake-action@v4 - with: - push: true - set: | - kubler.tags=ghcr.io/${{ github.repository }} - - - name: 👷 Portage Date - id: portage_date - run: | - set -eux - PORTAGE_DATE=$(docker run --rm -e TERM=dumb -w $HOME/.kubler "${KUBLER_IMAGE}" kubler portage) - echo "portage_date=$PORTAGE_DATE" - echo "portage_date=$PORTAGE_DATE" >> $GITHUB_OUTPUT - - - name: 👷 Get STAGE3_DATE - id: stage3_date - run: | - set -eux - docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" grep '^STAGE3_DATE=' builder/bob/build.conf - docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" grep '^STAGE3_DATE=' builder/bob-musl/build.conf - bob=$(docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" sed -n "s/^STAGE3_DATE='\(202[34][01][0-9]\{3\}T[0-9]\{6\}Z\)'$/\\1/p" builder/bob/build.conf) - bob_musl=$(docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" sed -n "s/^STAGE3_DATE='\(202[34][01][0-9]\{3\}T[0-9]\{6\}Z\)'$/\\1/p" builder/bob-musl/build.conf) - if [[ ("$bob" != "$bob_musl") ]]; then - echo "WARNING: bob and bob-musl have different STAGE3_DATE" - fi - echo "bob_stage3_date=$bob" >> $GITHUB_OUTPUT - echo "bob_musl_stage3_date=$bob_musl" >> $GITHUB_OUTPUT - - kubler-portage: - runs-on: ubuntu-latest - needs: setup - outputs: - portage_image: ${{ steps.portage_image.outputs.portage_image }} - env: - KUBLER_IMAGE: ghcr.io/${{ github.repository }} - PORTAGE_DATE: ${{ needs.setup.outputs.portage_date }} - steps: - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:latest - platforms: all - - - name: 🐋 Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - with: - # This breaks kubler https://github.com/edannenberg/kubler/issues/215 - # Sets up `docker build` command as an alias to `docker buildx` (default `false`) - install: true - - - name: 🐋 Docker Login - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: oras-project/setup-oras@v1 - - - run: | - oras version - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: 🐋🍳 Docker Bake Kubler portage - set PORTAGE_IMAGE env vars - run: | - set -eux - TMP_PORTAGE_IMAGE=ghcr.io/"${GITHUB_REPOSITORY}"/tmp/portage:"${PORTAGE_DATE}" - PORTAGE_IMAGE=ghcr.io/"${GITHUB_REPOSITORY}"/portage:"${PORTAGE_DATE}" - echo "TMP_PORTAGE_IMAGE=$TMP_PORTAGE_IMAGE" >> $GITHUB_ENV - echo "PORTAGE_IMAGE=$PORTAGE_IMAGE" >> $GITHUB_ENV - - - name: 🐋 Docker Pull Portage - id: portage-pull - continue-on-error: true - run: | - set -eu - docker pull "$PORTAGE_IMAGE" - - # We always test, even if we pulled - # A pushed image should have been tested before being pushed. - # But re-testing for extra safety and in case tests have changed in between. - - name: 🐋🍳 Docker Bake Kubler portage - test pulled - id: portage-pull-test - if: ${{ steps.portage-pull.outcome == 'success' }} - continue-on-error: true - run: | - set -eux - docker run --rm "${PORTAGE_IMAGE}" grep TIMESTAMP /var/db/repos/gentoo/Manifest - cd bob-portage || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${PORTAGE_IMAGE}" validate --color - - - name: 🐋🍳 Docker Bake Kubler portage - list targets - if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} - uses: docker/bake-action/subaction/list-targets@v4 - with: - workdir: bob-portage - - - name: 🐋🍳 Docker Bake Kubler portage - if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} - uses: docker/bake-action@v4 - with: - workdir: bob-portage - push: true - set: | - kubler-portage.tags=${{ env.TMP_PORTAGE_IMAGE }} - kubler-portage.cache-from=type=gha,scope=portage - kubler-portage.cache-to=type=gha,scope=portage - - - name: 🐋🍳 Docker Bake Kubler portage - test baked - if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} - run: | - set -eux - docker run --rm "${TMP_PORTAGE_IMAGE}" grep TIMESTAMP /var/db/repos/gentoo/Manifest - cd bob-portage || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${TMP_PORTAGE_IMAGE}" validate --color - - - name: 🐋🍳 Docker Bake Kubler portage - oras cp - if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} - run: | - set -eux - oras cp -v "$TMP_PORTAGE_IMAGE" "$PORTAGE_IMAGE" - - - name: Output PORTAGE_IMAGE - id: portage_image - run: | - set -eux - echo "portage_image=${PORTAGE_IMAGE}" >> $GITHUB_OUTPUT - - stage3: - runs-on: ubuntu-latest - needs: - - setup - - kubler-portage - env: - KUBLER_IMAGE: ghcr.io/${{ github.repository }} - PORTAGE_DATE: ${{ needs.setup.outputs.portage_date }} - PORTAGE_IMAGE: ${{ needs.kubler-portage.outputs.portage_image }} - strategy: - fail-fast: false - matrix: - include: - - name: bob - stage3: - base: gentoo/stage3:hardened-nomultilib - tmp_image: ghcr.io/${{ github.repository }}/tmp/stage3-amd64-hardened-nomultilib-openrc:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/stage3-amd64-hardened-nomultilib-openrc:${{ needs.setup.outputs.portage_date }} - cache: stage3-amd64-hardened-nomultilib-openrc - goss: stage3/amd64-hardened-nomultilib-openrc - core: - def_chost: x86_64-pc-linux-gnu - tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-core:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/bob-core:${{ needs.setup.outputs.portage_date }} - cache: bob-core - goss: core/bob-core - bob: - target: bob - tmp_image: ghcr.io/${{ github.repository }}/tmp/bob:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/bob:${{ needs.setup.outputs.portage_date }} - cache: bob - goss: builder/bob - - name: bob-musl - stage3: - base: gentoo/stage3:musl-hardened - tmp_image: ghcr.io/${{ github.repository }}/tmp/stage3-amd64-musl-hardened:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/stage3-amd64-musl-hardened:${{ needs.setup.outputs.portage_date }} - cache: stage3-amd64-musl-hardened - goss: stage3/amd64-musl-hardened - core: - def_chost: x86_64-gentoo-linux-musl - tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-musl-core:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/bob-musl-core:${{ needs.setup.outputs.portage_date }} - cache: bob-musl-core - goss: core/bob-musl-core - bob: - target: bob-musl - tmp_image: ghcr.io/${{ github.repository }}/tmp/bob-musl:${{ needs.setup.outputs.portage_date }} - image: ghcr.io/${{ github.repository }}/bob-musl:${{ needs.setup.outputs.portage_date }} - cache: bob-musl - goss: builder/bob-musl - steps: - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:latest - platforms: all - - - name: 🐋 Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - with: - # This breaks kubler https://github.com/edannenberg/kubler/issues/215 - # Sets up `docker build` command as an alias to `docker buildx` (default `false`) - install: true - - - name: 🐋 Docker Login - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Check out repository code - uses: actions/checkout@v4 - - - uses: oras-project/setup-oras@v1 - - - name: 🐋 Docker Pull stage3 - id: stage3-pull - continue-on-error: true - run: | - set -eu - docker pull "${{ matrix.stage3.image }}" - - - name: 🐋🍳 Docker Bake Kubler stage3 - test pulled - id: stage3-pull-test - if: ${{ steps.stage3-pull.outcome == 'success' }} - continue-on-error: true - run: | - set -eux - docker run --rm "${{ matrix.stage3.image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.stage3.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.stage3.image }}" eselect profile show - cd "${{ matrix.stage3.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.stage3.image }}" validate --color - - - name: 🐋🍳 Docker Bake Kubler stage3 - list targets - if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} - uses: docker/bake-action/subaction/list-targets@v4 - with: - workdir: bob-stage3 - - - name: 🐋🍳 Docker Bake Kubler stage3 - if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} - uses: docker/bake-action@v4 - with: - workdir: bob-stage3 - push: true - # In the `docker-bake.hcl` - # There is a variable `BASE_TAG` and `BASE_IMAGE`. `BASE_IMAGE` interpolates `BASE_TAG`i - # These can be set with env vars when using `docker buildx bake` command - # For the gentoo-stage3 target there's no `BASE_TAG` arg, only `BASE_IMAGE` which defaults to `BASE_IMAGE` var - # When using bake-action we need to set the `BASE_IMAGE` arg, setting `BASE_TAG` arg won't do what we want. - set: | - gentoo-stage3.args.BASE_IMAGE=${{ matrix.stage3.base }} - gentoo-stage3.args.PORTAGE=${{ env.PORTAGE_IMAGE }} - gentoo-stage3.tags=${{ matrix.stage3.tmp_image }} - gentoo-stage3.cache-from=type=gha,scope=portage - gentoo-stage3.cache-from=type=gha,scope=${{ matrix.stage3.cache }} - gentoo-stage3.cache-to=type=gha,scope=${{ matrix.stage3.cache }} - - - name: 🐋🍳 Docker Bake Kubler stage3 - test baked - if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} - run: | - set -eux - docker run --rm "${{ matrix.stage3.tmp_image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.stage3.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.stage3.tmp_image }}" eselect profile show - cd "${{ matrix.stage3.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.stage3.tmp_image }}" validate --color - - - name: 🐋🍳 Docker Bake Kubler stage3 - oras cp - if: ${{ steps.stage3-pull.outcome == 'failure' || steps.stage3-pull-test.outcome == 'failure' }} - run: | - set -eux - oras cp -v "${{ matrix.stage3.tmp_image }}" "${{ matrix.stage3.image }}" - - - name: 🐋 Docker Pull bob-core - id: bob-core-pull - continue-on-error: true - run: | - set -eu - docker pull "${{ matrix.core.image }}" - - - name: 🐋🍳 Docker Bake Kubler bob-core - test pulled - id: bob-core-pull-test - if: ${{ steps.bob-core-pull.outcome == 'success' }} - run: | - set -eux - docker run --rm "${{ matrix.core.image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.core.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.core.image }}" eselect profile show - docker run --rm "${{ matrix.core.image }}" ls -l /var/cache - cd "${{ matrix.core.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.core.image }}" validate --color - - - name: 🐋🍳 Docker Bake Kubler bob-core - list targets - if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} - uses: docker/bake-action/subaction/list-targets@v4 - with: - workdir: bob-core - - - name: 🐋🍳 Docker Bake Kubler bob-core - if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} - uses: docker/bake-action@v4 - with: - workdir: bob-core - push: true - set: | - core.args.DEF_CHOST=${{ matrix.core.def_chost }} - core.args.BASE_IMAGE=${{ matrix.stage3.image }} - core.tags=${{ matrix.core.tmp_image }} - core.cache-from=type=gha,scope=portage - core.cache-from=type=gha,scope=${{ matrix.stage3.cache }} - core.cache-from=type=gha,scope=${{ matrix.core.cache }} - core.cache-to=type=gha,scope=${{ matrix.core.cache }} - - - name: 🐋🍳 Docker Bake Kubler bob-core - test baked - if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} - run: | - set -eux - docker run --rm "${{ matrix.core.tmp_image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.core.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.core.tmp_image }}" eselect profile show - docker run --rm "${{ matrix.core.tmp_image }}" ls -l /var/cache - cd "${{ matrix.core.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.core.tmp_image }}" validate --color - - - name: 🐋🍳 Docker Bake Kubler bob-core - oras cp - if: ${{ steps.bob-core-pull.outcome == 'failure' || steps.bob-core-pull-test.outcome == 'failure' }} - run: | - set -eux - oras cp -v "${{ matrix.core.tmp_image }}" "${{ matrix.core.image }}" - - - name: 🐋 Docker Pull bob - id: bob-pull - continue-on-error: true - run: | - set -eu - docker pull "${{ matrix.bob.image }}" - - - name: 🐋 Docker Build kubler builder - bob - test pulled - id: bob-pull-test - if: ${{ steps.bob-pull.outcome == 'success' }} - run: | - set -eux - docker run --rm "${{ matrix.bob.image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.bob.image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.bob.image }}" eselect profile show - docker run --rm "${{ matrix.bob.image }}" ls -l /var/cache - docker run --rm "${{ matrix.bob.image }}" ls -l /var/cache/eix - docker run --rm "${{ matrix.bob.image }}" eix --selected -c - cd "${{ matrix.bob.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.bob.image }}" validate --color - - - name: 🐋 Docker Build kubler builder - bob - list targets - if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} - uses: docker/bake-action/subaction/list-targets@v4 - with: - workdir: ${{ matrix.bob.goss }} - - - name: 🐋🍳 Docker Bake Kubler builder - bob - if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} - uses: docker/bake-action@v4 - with: - workdir: ${{ matrix.bob.goss }} - push: true - set: | - ${{matrix.bob.target}}.args.BASE_IMAGE=${{ matrix.core.image }} - ${{matrix.bob.target}}.tags=${{ matrix.bob.tmp_image }} - ${{matrix.bob.target}}.cache-from=type=gha,scope=portage - ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.stage3.cache }} - ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.core.cache }} - ${{matrix.bob.target}}.cache-from=type=gha,scope=${{ matrix.bob.cache }} - ${{matrix.bob.target}}.cache-to=type=gha,scope=${{ matrix.bob.cache }} - - - name: 🐋 Docker Build kubler builder - bob - test baked - if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} - run: | - set -eux - docker run --rm "${{ matrix.bob.tmp_image }}" cat /etc/gentoo-release - docker run --rm "${{ matrix.bob.tmp_image }}" sh -c "grep -E 'Latest|stage3' /latest-stage3*.txt" - docker run --rm "${{ matrix.bob.tmp_image }}" eselect profile show - docker run --rm "${{ matrix.bob.tmp_image }}" ls -l /var/cache - docker run --rm "${{ matrix.bob.tmp_image }}" ls -l /var/cache/eix - docker run --rm "${{ matrix.bob.tmp_image }}" eix --selected -c - cd "${{ matrix.bob.goss }}" || exit 1 - docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "${KUBLER_IMAGE}" dgoss run -w /goss --entrypoint /goss/goss "${{ matrix.bob.tmp_image }}" validate --color - - - name: 🐋🍳 Docker Bake Kubler builder - bob - oras cp - if: ${{ steps.bob-pull.outcome == 'failure' || steps.bob-pull-test.outcome == 'failure' }} - run: | - set -eux - oras cp -v "${{ matrix.bob.tmp_image }}" "${{ matrix.bob.image }}" - kubler: needs: - setup diff --git a/.github/workflows/portage.yml b/.github/workflows/portage.yml new file mode 100644 index 0000000..00612f4 --- /dev/null +++ b/.github/workflows/portage.yml @@ -0,0 +1,114 @@ +name: Portage +on: + #push: + # branches: + # - main + workflow_dispatch: + workflow_call: + +# Jobs run in parallel +# Jobs are independent with separate file systems, IP addresses, etc. +jobs: + portage: + runs-on: ubuntu-latest + needs: setup + outputs: + portage_image: ${{ steps.portage_image.outputs.portage_image }} + env: + KUBLER_IMAGE: ghcr.io/${{ github.repository }} + PORTAGE_DATE: ${{ needs.setup.outputs.portage_date }} + steps: + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:latest + platforms: all + + - name: 🐋 Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + # This breaks kubler https://github.com/edannenberg/kubler/issues/215 + # Sets up `docker build` command as an alias to `docker buildx` (default `false`) + install: true + + - name: 🐋 Docker Login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: oras-project/setup-oras@v1 + + - run: | + oras version + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: 🐋🍳 Docker Bake Kubler portage - set PORTAGE_IMAGE env vars + run: | + set -eux + TMP_PORTAGE_IMAGE=ghcr.io/"${GITHUB_REPOSITORY}"/tmp/portage:"${PORTAGE_DATE}" + PORTAGE_IMAGE=ghcr.io/"${GITHUB_REPOSITORY}"/portage:"${PORTAGE_DATE}" + echo "TMP_PORTAGE_IMAGE=$TMP_PORTAGE_IMAGE" >> $GITHUB_ENV + echo "PORTAGE_IMAGE=$PORTAGE_IMAGE" >> $GITHUB_ENV + + - name: 🐋 Docker Pull Portage + id: portage-pull + continue-on-error: true + run: | + set -eu + docker pull "$PORTAGE_IMAGE" + + # We always test, even if we pulled + # A pushed image should have been tested before being pushed. + # But re-testing for extra safety and in case tests have changed in between. + - name: 🐋🍳 Docker Bake Kubler portage - test pulled + id: portage-pull-test + if: ${{ steps.portage-pull.outcome == 'success' }} + continue-on-error: true + run: | + set -eux + docker run --rm "${PORTAGE_IMAGE}" grep TIMESTAMP /var/db/repos/gentoo/Manifest + cd bob-portage || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${PORTAGE_IMAGE}" validate --color + + - name: 🐋🍳 Docker Bake Kubler portage - list targets + if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} + uses: docker/bake-action/subaction/list-targets@v4 + with: + workdir: bob-portage + + - name: 🐋🍳 Docker Bake Kubler portage + if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} + uses: docker/bake-action@v4 + with: + workdir: bob-portage + push: true + set: | + kubler-portage.tags=${{ env.TMP_PORTAGE_IMAGE }} + kubler-portage.cache-from=type=gha,scope=portage + kubler-portage.cache-to=type=gha,scope=portage + + - name: 🐋🍳 Docker Bake Kubler portage - test baked + if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} + run: | + set -eux + docker run --rm "${TMP_PORTAGE_IMAGE}" grep TIMESTAMP /var/db/repos/gentoo/Manifest + cd bob-portage || exit 1 + docker run --rm -v /run/docker.sock:/run/docker.sock -v "$(pwd):/src:ro" -w /src -e CONTAINER_MODE=entrypoint "$KUBLER_IMAGE" dgoss run -w /goss --entrypoint /goss/goss "${TMP_PORTAGE_IMAGE}" validate --color + + - name: 🐋🍳 Docker Bake Kubler portage - oras cp + if: ${{ steps.portage-pull.outcome == 'failure' || steps.portage-pull-test.outcome == 'failure' }} + run: | + set -eux + oras cp -v "$TMP_PORTAGE_IMAGE" "$PORTAGE_IMAGE" + + - name: Output PORTAGE_IMAGE + id: portage_image + run: | + set -eux + echo "portage_image=${PORTAGE_IMAGE}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml new file mode 100644 index 0000000..270aa8d --- /dev/null +++ b/.github/workflows/setup.yml @@ -0,0 +1,100 @@ +# Bakes a docker image for running kubler in later jobs +name: Bake Kubler +on: + #push: + # branches: + # - main + workflow_dispatch: + workflow_call: + +# Jobs run in parallel +# Jobs are independent with separate file systems, IP addresses, etc. +jobs: + setup: + name: Setup Kubler + runs-on: ubuntu-latest + outputs: + portage_date: ${{ steps.portage_date.outputs.portage_date }} + bob_stage3_date: ${{ steps.stage3.outputs.bob_stage3_date }} }} + bob_musl_stage3_date: ${{ steps.stage3.outputs.bob_musl_stage3_date }} }} + env: + KUBLER_IMAGE: ghcr.io/${{ github.repository }} + steps: + - name: inspect + run: | + set -eux + echo "${{ github.workspace }}" + ls -la "${{ github.workspace }}" + type systemctl + which systemctl + file $(which systemctl) + ldd $(which systemctl) + ls -la /var/run/dbus/system_bus_socket || true + pidof systemd || true + ps -ef + ps -ef --forest || true + id + uname -a + pwd + echo $HOME + env | grep USER + cd + pwd + mount + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:latest + platforms: all + + - name: 🐋 Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + # This breaks kubler https://github.com/edannenberg/kubler/issues/215 + # Sets up `docker build` command as an alias to `docker buildx` (default `false`) + install: true + + - name: 🐋 Docker Login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + # this is the default, end of job will logout + #logout: true + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: 🐋🍳 Docker Bake Kubler + #if: ${{ steps.portage-pull.outcome == 'failure' }} + uses: docker/bake-action@v4 + with: + push: true + set: | + kubler.tags=ghcr.io/${{ github.repository }} + + - name: 👷 Portage Date + id: portage_date + run: | + set -eux + PORTAGE_DATE=$(docker run --rm -e TERM=dumb -w $HOME/.kubler "${KUBLER_IMAGE}" kubler portage) + echo "portage_date=$PORTAGE_DATE" + echo "portage_date=$PORTAGE_DATE" >> $GITHUB_OUTPUT + + - name: 👷 Get STAGE3_DATE + id: stage3_date + run: | + set -eux + docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" grep '^STAGE3_DATE=' builder/bob/build.conf + docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" grep '^STAGE3_DATE=' builder/bob-musl/build.conf + bob=$(docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" sed -n "s/^STAGE3_DATE='\(202[34][01][0-9]\{3\}T[0-9]\{6\}Z\)'$/\\1/p" builder/bob/build.conf) + bob_musl=$(docker run --rm -w $HOME/.kubler/namespaces/kubler "${KUBLER_IMAGE}" sed -n "s/^STAGE3_DATE='\(202[34][01][0-9]\{3\}T[0-9]\{6\}Z\)'$/\\1/p" builder/bob-musl/build.conf) + if [[ ("$bob" != "$bob_musl") ]]; then + echo "WARNING: bob and bob-musl have different STAGE3_DATE" + fi + echo "bob_stage3_date=$bob" >> $GITHUB_OUTPUT + echo "bob_musl_stage3_date=$bob_musl" >> $GITHUB_OUTPUT