🧹 Speed up container image build process (#309)

* change build process for containers such that binaries are build on the host Signed-off-by: Ivan Milchev <[email protected]> * fixed version pin on cosign-installer action Signed-off-by: Ivan Milchev <[email protected]> * attempt to create a multi-arch container manifest Signed-off-by: Ivan Milchev <[email protected]> * extracted virtual tag creation to a script Signed-off-by: Ivan Milchev <[email protected]> * moved location of push-virtual-tag.sh Signed-off-by: Ivan Milchev <[email protected]> * added checkout step in push-virtual-tag Signed-off-by: Ivan Milchev <[email protected]> * use bash instead of sh Signed-off-by: Ivan Milchev <[email protected]> * added some logging Signed-off-by: Ivan Milchev <[email protected]> * set TAGS when pushing virtual tag Signed-off-by: Ivan Milchev <[email protected]> * simplified push-virtual-tag.sh Signed-off-by: Ivan Milchev <[email protected]> * added more comments Signed-off-by: Ivan Milchev <[email protected]> * fixed broken script Signed-off-by: Ivan Milchev <[email protected]> * move signing virtual tags into push-virtual-tag.sh Signed-off-by: Ivan Milchev <[email protected]> * add caching for Go Signed-off-by: Ivan Milchev <[email protected]> * fake change to check workflows execution times Signed-off-by: Ivan Milchev <[email protected]> * removed extra branch from publish workflow Signed-off-by: Ivan Milchev <[email protected]> * moved TARGET_OS=linux from load-minikube to docker-build Signed-off-by: Ivan Milchev <[email protected]> * increase retry loop in an attempt to make the integration tests more stable Signed-off-by: Ivan Milchev <[email protected]>
mondoohq · Apr 29, 2022 · a3286e1 · a3286e1
1 parent 26ffc7a
commit a3286e1
Show file tree

Hide file tree

Showing 7 changed files with 196 additions and 66 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,5 +1,4 @@
 # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
 # Ignore build and test binaries.
-bin/
 testbin/
 tests/
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -22,13 +22,19 @@ env:
 
 jobs:
   build-operator:
+    name: Build operator binaries
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
       # This is used to complete the identity challenge
       # with sigstore/fulcio when running outside of PRs.
       id-token: write
+
+    strategy:
+      matrix:
+        os: [linux]
+        arch: [amd64, arm64, arm]
 
     steps:
       - name: extract tag/version
@@ -38,27 +44,44 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      # Install the cosign tool except on PR
-      # https://github.com/sigstore/cosign-installer
-      - name: Install cosign
-        uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422
+      - uses: actions/setup-go@v2
         with:
-          cosign-release: "v1.4.0"
+          go-version: "${{ env.golang-version }}"
+
+      # Get values for cache paths to be used in later steps
+      - id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Checkout
+        uses: actions/checkout@v2
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      # Cache go build cache, used to speedup go test
+      - name: Go Build Cache
+        uses: actions/cache@v2
         with:
-          image: tonistiigi/binfmt:latest
-          platforms: amd64,arm
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-${{ matrix.os }}-${{ matrix.arch }}-go-build-${{ hashFiles('**/go.sum') }}
 
-      # Workaround: https://github.com/docker/build-push-action/issues/461
-      - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+      # Cache go mod cache, used to speedup builds
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-${{ matrix.os }}-${{ matrix.arch }}-go-mod-${{ hashFiles('**/go.sum') }}
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        uses: sigstore/[email protected]
+        with:
+          cosign-release: "v1.8.0"
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -68,23 +91,26 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v3
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-${{ matrix.arch }},onlatest=true
+
+      - name: Build binaries
+        run: VERSION=${{ steps.get_version.outputs.VERSION }} TARGET_OS=${{ matrix.os }} TARGET_ARCH=${{ matrix.arch }} make build
 
       # Build and push Docker image with Buildx
       # https://github.com/docker/build-push-action
       - name: Build and push operator image
         id: build-and-push-operator
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v2
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: ${{ matrix.os }}/${{ matrix.arch }}
           push: true
           labels: ${{ steps.meta.outputs.labels }}
           tags: ${{ steps.meta.outputs.tags }}
-          build_args: |
-            VERSION=${{ steps.get_version.outputs.VERSION }}
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
@@ -98,10 +124,57 @@ jobs:
         # against the sigstore community Fulcio instance.
         run: cosign sign ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push-operator.outputs.digest }}
 
+  push-virtual-tag:
+    name: Push multi-platform virtual tag
+    runs-on: ubuntu-latest
+    needs:
+      - build-operator
+
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        uses: sigstore/[email protected]
+        with:
+          cosign-release: "v1.8.0"
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Push multi-platform virtual tag and sign
+        run: bash scripts/push-virtual-tag.sh
+        env:
+          TAGS: ${{ steps.meta.outputs.tags }}
+          CPU_ARCHS: amd64 arm64 arm
+
   build-bundle:
     if: startsWith(github.ref, 'refs/tags/v')
     needs:
-      - build-operator
+      - push-virtual-tag
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -121,24 +194,14 @@ jobs:
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
       - name: Install cosign
-        uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422
-        with:
-          cosign-release: "v1.4.0"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: sigstore/[email protected]
         with:
-          image: tonistiigi/binfmt:latest
-          platforms: amd64,arm
-
-      # Workaround: https://github.com/docker/build-push-action/issues/461
-      - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+          cosign-release: "v1.8.0"
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -163,14 +226,14 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta-bundle
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v3
         with:
           images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-bundle"
 
       # Build and push Docker image bundle with Buildx
       - name: Build and push bundle image
         id: build-and-push-bundle
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v2
         with:
           context: .
           file: bundle.Dockerfile
@@ -205,11 +268,11 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/v')
     uses: ./.github/workflows/release-manifests.yaml
     needs:
-      - build-operator
+      - push-virtual-tag
 
   # publish helm chart after the release of container images is complete
   run-release-helm:
     if: startsWith(github.ref, 'refs/tags/v')
     uses: ./.github/workflows/release-helm-chart.yaml
     needs:
-      - build-operator
+      - push-virtual-tag
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -19,6 +19,30 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: "${{ env.golang-version }}"
+
+      # Get values for cache paths to be used in later steps
+      - id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)" 
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # Cache go build cache, used to speedup go test
+      - name: Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+
+      # Cache go mod cache, used to speedup builds
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
       - run: make test/ci
       - uses: actions/upload-artifact@v3  # upload test results
         if: success() || failure()        # run this step even if previous step failed
@@ -41,6 +65,30 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: "${{ env.golang-version }}"
+
+            # Get values for cache paths to be used in later steps
+      - id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # Cache go build cache, used to speedup go test
+      - name: Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+
+      # Cache go mod cache, used to speedup builds
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
       - name: Store creds
         run: |
           echo ${{ secrets.MONDOO_CLIENT }} | base64 -d > creds.json

diff --git a/Dockerfile b/Dockerfile
@@ -1,31 +1,9 @@
-# Build the manager binary
-FROM docker.io/library/golang:1.17 as builder
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
-COPY pkg/ pkg/
-
-# Build
-ARG VERSION
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -a -o manager -ldflags "-X go.mondoo.com/mondoo-operator/pkg/version.Version=$VERSION" main.go
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -a -o webhook pkg/webhooks/main.go
-
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/webhook .
+ADD bin/manager .
+ADD bin/webhook .
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]
diff --git a/Makefile b/Makefile
@@ -57,6 +57,10 @@ SHELL = /usr/bin/env bash -o pipefail
 # List all packages that contain unit tests. Ignore the integration tests folder.
 UNIT_TEST_PACKAGES=$(shell go list ./... | grep -v /tests/integration)
 
+# The target architecture for the binaries to be built.
+TARGET_ARCH?=$(shell go env GOARCH)
+TARGET_OS?=$(shell go env GOOS)
+
 all: build
 
 ##@ General
@@ -108,14 +112,15 @@ test/integration/ci: manifests generate generate-manifests load-minikube gotests
 ##@ Build
 
 build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager -ldflags "-s -w -X go.mondoo.com/mondoo-operator/pkg/version.Version=${VERSION}" main.go
-	go build -o bin/webhook pkg/webhooks/main.go
+	CGO_ENABLED=0 GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -o bin/manager -ldflags "-s -w -X go.mondoo.com/mondoo-operator/pkg/version.Version=${VERSION}" main.go
+	CGO_ENABLED=0 GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -o bin/webhook pkg/webhooks/main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	MONDOO_OPERATOR_NAMESPACE=mondoo-operator go run ./main.go
 
-docker-build: ## Build docker image with the manager.
-	docker build --build-arg VERSION=${VERSION} -t ${IMG} .
+docker-build: TARGET_OS=linux
+docker-build: build ## Build docker image with the manager.
+	docker build --platform=$(TARGET_ARCH) -t ${IMG} .
 
 load-minikube: docker-build ## Build docker image with the manager and load it into minikube.
 	minikube image load ${IMG}

diff --git a/scripts/push-virtual-tag.sh b/scripts/push-virtual-tag.sh
@@ -0,0 +1,37 @@
+# exit immediately when a command fails
+set -e
+# only exit with zero if all commands of the pipeline exit successfully
+set -o pipefail
+
+export COSIGN_EXPERIMENTAL=true
+export DOCKER_CLI_EXPERIMENTAL=enabled
+for tag in ${TAGS}; do
+    # Create manifest to join all images under one virtual tag
+    docker manifest create -a "$tag" \
+            "$tag-amd64" \
+            "$tag-arm64" \
+            "$tag-arm"
+    echo "Created manifest list $tag"
+
+    # Annotate to set which image is build for which CPU architecture
+    for arch in ${CPU_ARCHS}; do
+        docker manifest annotate --arch "$arch" "$tag" "$tag-$arch"
+    done    
+    echo "Pushing manifest list $tag..."
+    DIGEST=$(docker manifest push "$tag")
+    echo "Pushed manifest list $tag"
+    echo "Signing digest $DIGEST"
+
+    # Sign the resulting Docker image digest except on PRs.
+    # This will only write to the public Rekor transparency log when the Docker
+    # repository is public to avoid leaking data.  If you would like to publish
+    # transparency data even for private images, pass --force to cosign below.
+    # https://github.com/sigstore/cosign
+
+    # This step uses the identity token to provision an ephemeral certificate
+    # against the sigstore community Fulcio instance.
+
+    # Remove the tag from the image and append the digest instead.
+    cosign sign "${tag%:*}@$DIGEST"
+    echo "Digest $DIGEST signed"
+done