From e578177577e9d0c93c3e66f6c2f19f09986e8982 Mon Sep 17 00:00:00 2001 From: Aidan Date: Tue, 12 Nov 2024 15:52:48 +0800 Subject: [PATCH] ci: extend build process --- .github/workflows/release.yaml | 200 +++++++++++++++++++++++++++++---- Justfile | 26 +++-- 2 files changed, 194 insertions(+), 32 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2e6ab42d0..f92d44fd0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,139 +28,299 @@ jobs: build_and_release_mater_cli: runs-on: self-hosted + permissions: + contents: write + packages: write + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'mater-cli-v') steps: - uses: actions/checkout@v4.2.2 - - name: Check version match + - id: check-version + name: Check version match run: | + # used later in the build for docker tagging PACKAGE_VERSION="$(cargo metadata --no-deps --color never --format-version 1 --locked | - jq -r '.packages[] | select(.name == "mater-cli") | "\(.name)-v\(.version)"')" - if [[ "$PACKAGE_VERSION" != "$GITHUB_REF_NAME" ]]; then + jq -r '.packages[] | select(.name == "mater-cli") | .version')" + + # used for checking the tag + PACKAGE_VERSION_WITH_NAME=$(echo "mater-cli-v$PACKAGE_VERSION") + + if [[ "$PACKAGE_VERSION_WITH_NAME" != "$GITHUB_REF_NAME" ]]; then exit 1; fi + # make the version available as a "checked" output in later steps + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT" + - name: Build in release mode run: RUSTFLAGS="-D warnings" cargo build --release --locked --package mater-cli - - name: Release binaries + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish image + run: | + docker build \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --tag ghcr.io/eigerco/mater-cli:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" \ + --file ./docker/dockerfiles/mater-cli.Dockerfile \ + . + docker push ghcr.io/eigerco/mater-cli:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" + + - name: Perform release uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # set github token for rw rights on repo with: tag_name: ${{ github.ref_name }} # set the name of the release the tag + body: | + Docker container release: https://github.com/eigerco/polka-storage/pkgs/container/mater-cli/?tag=${{ steps.check-version.outputs.PACKAGE_VERSION }} files: | target/release/mater-cli build_and_release_storagext_cli: runs-on: self-hosted + permissions: + contents: write + packages: write + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'storagext-cli-v') steps: - uses: actions/checkout@v4.2.2 - - name: Check version match + - id: check-version + name: Check version match run: | + # used later in the build for docker tagging PACKAGE_VERSION="$(cargo metadata --no-deps --color never --format-version 1 --locked | - jq -r '.packages[] | select(.name == "storagext-cli") | "\(.name)-v\(.version)"')" - if [[ "$PACKAGE_VERSION" != "$GITHUB_REF_NAME" ]]; then + jq -r '.packages[] | select(.name == "storagext-cli") | .version')" + + # used for checking the tag + PACKAGE_VERSION_WITH_NAME=$(echo "storagext-cli-v$PACKAGE_VERSION") + + if [[ "$PACKAGE_VERSION_WITH_NAME" != "$GITHUB_REF_NAME" ]]; then exit 1; fi + # make the version available as a "checked" output in later steps + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT" + - name: Build in release mode run: RUSTFLAGS="-D warnings" cargo build --release --locked --package storagext-cli - - name: Release binaries + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish image + run: | + docker build \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --tag ghcr.io/eigerco/storagext-cli:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" \ + --file ./docker/dockerfiles/storagext-cli.Dockerfile \ + . + docker push ghcr.io/eigerco/storagext-cli:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" + + - name: Perform release uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # set github token for rw rights on repo with: tag_name: ${{ github.ref_name }} # set the name of the release the tag + body: | + Docker container release: https://github.com/eigerco/polka-storage/pkgs/container/storagext-cli/?tag=${{ steps.check-version.outputs.PACKAGE_VERSION }} files: | target/release/storagext-cli build_and_release_polka_storage_node: runs-on: self-hosted + permissions: + contents: write + packages: write + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'polka-storage-node-v') steps: - uses: actions/checkout@v4.2.2 - - name: Check version match + - id: check-version + name: Check version match run: | + # used later in the build for docker tagging PACKAGE_VERSION="$(cargo metadata --no-deps --color never --format-version 1 --locked | - jq -r '.packages[] | select(.name == "polka-storage-node") | "\(.name)-v\(.version)"')" - if [[ "$PACKAGE_VERSION" != "$GITHUB_REF_NAME" ]]; then + jq -r '.packages[] | select(.name == "polka-storage-node") | .version')" + + # used for checking the tag + PACKAGE_VERSION_WITH_NAME=$(echo "polka-storage-node-v$PACKAGE_VERSION") + + if [[ "$PACKAGE_VERSION_WITH_NAME" != "$GITHUB_REF_NAME" ]]; then exit 1; fi + # make the version available as a "checked" output in later steps + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT" + - name: Build in release mode run: RUSTFLAGS="-D warnings" cargo build --release --locked --package polka-storage-node - - name: Release binaries + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish image + run: | + docker build \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --tag ghcr.io/eigerco/polka-storage-node:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" \ + --file ./docker/dockerfiles/polka-storage-node.Dockerfile \ + . + docker push ghcr.io/eigerco/polka-storage-node:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" + + - name: Perform release uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # set github token for rw rights on repo with: tag_name: ${{ github.ref_name }} # set the name of the release the tag + body: | + Docker container release: https://github.com/eigerco/polka-storage/pkgs/container/polka-storage-node/?tag=${{ steps.check-version.outputs.PACKAGE_VERSION }} files: | target/release/polka-storage-node build_and_release_polka_storage_provider_server: runs-on: self-hosted + permissions: + contents: write + packages: write + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'polka-storage-provider-server-v') steps: - uses: actions/checkout@v4.2.2 - - name: Check version match + - id: check-version + name: Check version match run: | + # used later in the build for docker tagging PACKAGE_VERSION="$(cargo metadata --no-deps --color never --format-version 1 --locked | - jq -r '.packages[] | select(.name == "polka-storage-server") | "\(.name)-v\(.version)"')" - if [[ "$PACKAGE_VERSION" != "$GITHUB_REF_NAME" ]]; then + jq -r '.packages[] | select(.name == "polka-storage-provider-server") | .version')" + + # used for checking the tag + PACKAGE_VERSION_WITH_NAME=$(echo "polka-storage-provider-server-v$PACKAGE_VERSION") + + if [[ "$PACKAGE_VERSION_WITH_NAME" != "$GITHUB_REF_NAME" ]]; then exit 1; fi + # make the version available as a "checked" output in later steps + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT" + - name: Build in release mode run: RUSTFLAGS="-D warnings" cargo build --release --locked --package polka-storage-provider-server - - name: Release binaries + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish image + run: | + docker build \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --tag ghcr.io/eigerco/polka-storage-provider-server:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" \ + --file ./docker/dockerfiles/polka-storage-provider-server.Dockerfile \ + . + docker push ghcr.io/eigerco/polka-storage-provider-server:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" + + - name: Perform release uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # set github token for rw rights on repo with: tag_name: ${{ github.ref_name }} # set the name of the release the tag + body: | + Docker container release: https://github.com/eigerco/polka-storage/pkgs/container/polka-storage-provider-server/?tag=${{ steps.check-version.outputs.PACKAGE_VERSION }} files: | target/release/polka-storage-provider-server build_and_release_polka_storage_provider_client: runs-on: self-hosted + permissions: + contents: write + packages: write + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'polka-storage-provider-client-v') steps: - uses: actions/checkout@v4.2.2 - - name: Check version match + - id: check-version + name: Check version match run: | + # used later in the build for docker tagging PACKAGE_VERSION="$(cargo metadata --no-deps --color never --format-version 1 --locked | - jq -r '.packages[] | select(.name == "polka-storage-client") | "\(.name)-v\(.version)"')" - if [[ "$PACKAGE_VERSION" != "$GITHUB_REF_NAME" ]]; then + jq -r '.packages[] | select(.name == "polka-storage-provider-client") | .version')" + + # used for checking the tag + PACKAGE_VERSION_WITH_NAME=$(echo "polka-storage-provider-client-v$PACKAGE_VERSION") + + if [[ "$PACKAGE_VERSION_WITH_NAME" != "$GITHUB_REF_NAME" ]]; then exit 1; fi + # make the version available as a "checked" output in later steps + echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT" + - name: Build in release mode run: RUSTFLAGS="-D warnings" cargo build --release --locked --package polka-storage-provider-client - - name: Release binaries + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish image + run: | + docker build \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --tag ghcr.io/eigerco/polka-storage-provider-client:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" \ + --file ./docker/dockerfiles/polka-storage-provider-client.Dockerfile \ + . + docker push ghcr.io/eigerco/polka-storage-provider-client:"${{ steps.check-version.outputs.PACKAGE_VERSION }}" + + - name: Perform release uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # set github token for rw rights on repo with: tag_name: ${{ github.ref_name }} # set the name of the release the tag + body: | + Docker container release: https://github.com/eigerco/polka-storage/pkgs/container/polka-storage-provider-client/?tag=${{ steps.check-version.outputs.PACKAGE_VERSION }} files: | target/release/polka-storage-provider-client diff --git a/Justfile b/Justfile index 29b794230..31a5bd3b0 100644 --- a/Justfile +++ b/Justfile @@ -65,15 +65,17 @@ build-mater-cli: # Build all the binaries build-binaries-all: build-polka-storage-node build-polka-storage-provider-client build-polka-storage-provider-server build-storagext-cli build-mater-cli -# TODO: the docker builds point to the azure container registry — they shouldn't -# we should be using the GHCR this time + +# NOTE: Docker builds have no ghcr prefix because these are built locally. +# The Docker images built in the CI will point to the ghcr. +# Done to differenciate between local images and pulled images. # Build the mater CLI binary build-mater-docker: docker build \ --build-arg VCS_REF="$(git rev-parse HEAD)" \ --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ - -t polkadotstorage.azurecr.io/mater-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ + -t mater-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ --file ./docker/dockerfiles/mater-cli.Dockerfile \ . @@ -82,7 +84,7 @@ build-polka-storage-node-docker: docker build \ --build-arg VCS_REF="$(git rev-parse HEAD)" \ --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ - -t polkadotstorage.azurecr.io/polka-storage-node:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ + -t polka-storage-node:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ --file ./docker/dockerfiles/polka-storage-node.Dockerfile \ . @@ -91,7 +93,7 @@ build-polka-storage-provider-client-docker: docker build \ --build-arg VCS_REF="$(git rev-parse HEAD)" \ --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ - -t polkadotstorage.azurecr.io/polka-storage-provider-client:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ + -t polka-storage-provider-client:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ --file ./docker/dockerfiles/polka-storage-provider-client.Dockerfile \ . @@ -100,7 +102,7 @@ build-polka-storage-provider-server-docker: docker build \ --build-arg VCS_REF="$(git rev-parse HEAD)" \ --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ - -t polkadotstorage.azurecr.io/polka-storage-provider-server:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ + -t polka-storage-provider-server:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ --file ./docker/dockerfiles/polka-storage-provider-server.Dockerfile \ . @@ -109,7 +111,7 @@ build-storagext-docker: docker build \ --build-arg VCS_REF="$(git rev-parse HEAD)" \ --build-arg BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ - -t polkadotstorage.azurecr.io/storagext-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ + -t storagext-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" \ --file ./docker/dockerfiles/storagext-cli.Dockerfile \ . @@ -120,27 +122,27 @@ build-docker-all: build-polka-storage-node-docker build-polka-storage-provider-c # Run the mater CLI docker image # This only works if the image is already built run-mater-docker: - docker run -it polkadotstorage.azurecr.io/mater-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" + docker run -it mater-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" # Run the parachain node docker image # This only works if the image is already built run-polka-storage-node-docker: - docker run -it polkadotstorage.azurecr.io/parachain-node:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" + docker run -it parachain-node:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" # Run the storage provider client docker image # This only works if the image is already built run-polka-storage-client-docker: - docker run -it polkadotstorage.azurecr.io/sp-client:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" + docker run -it sp-client:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" # Run the storage provider server docker image # This only works if the image is already built run-polka-storage-server-docker: - docker run -it polkadotstorage.azurecr.io/sp-server:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" + docker run -it sp-server:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" # Run the storagext CLI docker image # This only works if the image is already built run-storagext-docker: - docker run -it polkadotstorage.azurecr.io/storagext-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" + docker run -it storagext-cli:"$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[0].version')" load-to-minikube: # https://github.com/paritytech/zombienet/pull/1830