From 90d2797d2f17a9898ebab7a9e3d1d44eb1e13839 Mon Sep 17 00:00:00 2001 From: Laurent Vaylet Date: Fri, 17 May 2024 16:35:12 +0200 Subject: [PATCH] build: make the Docker image smaller (#468) * stop copying everything! * clean up and improve overall Docker workflow * stop running Docker tests in CI * rename CI workflow * shave an extra 27MB with `pip install --no-cache-dir ...` * shave an extra 75MB with Alpine image * chore: add missing variables in sample environment file * stop running Docker tests in Cloud Build * clean up and reformat `Makefile` * clean up `Makefile` and GitHub Worflows --- .dockerignore | 40 ++++++++++ .github/workflows/build-and-push-to-gcr.yml | 6 +- .github/workflows/{test.yml => ci.yml} | 5 +- .github/workflows/deploy.yml | 6 +- Dockerfile | 37 ++++++--- Makefile | 87 ++++++++++----------- cloudbuild.yaml | 17 ++-- docs/deploy/cloudbuild.md | 2 +- 8 files changed, 121 insertions(+), 79 deletions(-) create mode 100644 .dockerignore rename .github/workflows/{test.yml => ci.yml} (96%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..58bd19a2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +# Git +.git +.gitignore +.gitattributes +.github + +# Docker +Dockerfile +.dockerignore + +# Byte-compiled / optimized / DLL files +**/__pycache__/ +**/*.py[cod] + +build/ +dist/ + +# Testing / Linting +.mypy_cache/ +.pytest_cache/ +.pytype/ +.ruff_cache/ +.coverage +tests/ + +# Documentation +docs/ + +# Configuration +.env + +# Virtual Environments +.venv/ +venv/ + +# VS Code +.vscode/ + +# Cloud Build +cloudbuild.yaml diff --git a/.github/workflows/build-and-push-to-gcr.yml b/.github/workflows/build-and-push-to-gcr.yml index debe85f3..a24b6044 100644 --- a/.github/workflows/build-and-push-to-gcr.yml +++ b/.github/workflows/build-and-push-to-gcr.yml @@ -9,7 +9,7 @@ on: - v*.*.* jobs: - cloudbuild: + cloud-build: runs-on: ubuntu-latest environment: prod steps: @@ -39,14 +39,14 @@ jobs: project_id: ${{ secrets.PROJECT_ID }} - name: Build Docker container and publish on GCR - run: make cloudbuild || true + run: make cloud_build || true env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} VERSION: ${{ steps.check-tag.outputs.match == 'true' && steps.check-tag.outputs.version || github.event.pull_request.head.sha || github.sha }} - name: Build Docker container and publish on GCR [latest] - run: make cloudbuild || true + run: make cloud_build || true if: ${{ steps.check-tag.outputs.match == 'true' }} env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yml similarity index 96% rename from .github/workflows/test.yml rename to .github/workflows/ci.yml index 294b1f92..f1e7eeee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: test +name: Continous Integration (CI) on: pull_request: @@ -74,6 +74,3 @@ jobs: - name: Build Docker image run: make docker_build - - - name: Run Docker tests - run: make docker_test diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cbb95b83..50b5c952 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,7 +9,7 @@ on: - v*.*.* jobs: - cloudrun: + cloud-run: runs-on: ubuntu-latest environment: prod concurrency: prod @@ -40,7 +40,7 @@ jobs: id: wait-build with: token: ${{ secrets.GITHUB_TOKEN }} - checkName: cloudbuild + checkName: cloud-build ref: ${{ github.event.pull_request.head.sha || github.sha }} intervalSeconds: 10 timeoutSeconds: 900 # 15m @@ -58,7 +58,7 @@ jobs: run: echo Timeout && false # fail if build time out - name: Deploy Docker container to Cloud Run - run: make cloudrun + run: make cloud_run env: GCR_PROJECT_ID: ${{ secrets.GCR_PROJECT_ID }} CLOUDBUILD_PROJECT_ID: ${{ secrets.CLOUDBUILD_PROJECT_ID }} diff --git a/Dockerfile b/Dockerfile index f308b85f..2b3a7a78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,16 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim -RUN apt-get update && \ - apt-get install -y \ - build-essential \ - make \ - gcc \ - locales -ADD . /app +ARG PYTHON_VERSION + +FROM python:${PYTHON_VERSION}-alpine + WORKDIR /app -RUN pip install -U setuptools -RUN pip install ."[api, datadog, dynatrace, prometheus, elasticsearch, opensearch, splunk, pubsub, cloud_monitoring, cloud_service_monitoring, cloud_storage, bigquery, cloudevent, dev]" + +COPY . ./ + +# TODO: Is `dev` required if we decide not to run tests from the Docker image? +RUN pip install --no-cache-dir ."[ \ + api, \ + bigquery, \ + cloud_monitoring, \ + cloud_service_monitoring, \ + cloud_storage, \ + cloudevent, \ + datadog, \ + dynatrace, \ + elasticsearch, \ + opensearch, \ + prometheus, \ + pubsub, \ + splunk \ + ]" + ENTRYPOINT [ "slo-generator" ] -CMD ["-v"] + +CMD [ "-v" ] diff --git a/Makefile b/Makefile index 18e390fa..28b9cc9f 100644 --- a/Makefile +++ b/Makefile @@ -130,53 +130,48 @@ int_os: run_api: slo-generator api --target=run_compute --signature-type=http -c samples/config.yaml -# Local Docker build / push +# Build Docker image locally docker_build: DOCKER_BUILDKIT=1 - docker build -t slo-generator:latest . - -docker_test: docker_build - docker run --entrypoint "make" \ - -e GOOGLE_APPLICATION_CREDENTIALS=tests/unit/fixtures/fake_credentials.json \ - slo-generator test - -# Cloudbuild -cloudbuild: gcloud_alpha - gcloud alpha builds submit \ - --config=cloudbuild.yaml \ - --project=${CLOUDBUILD_PROJECT_ID} \ - --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} - -# Cloudrun -cloudrun: + docker build \ + -t slo-generator:latest \ + --build-arg PYTHON_VERSION=3.9 \ + . + +# Build Docker image with Cloud Build +cloud_build: + gcloud builds submit \ + --config=cloudbuild.yaml \ + --project=${CLOUDBUILD_PROJECT_ID} \ + --substitutions=_GCR_PROJECT_ID=${GCR_PROJECT_ID},_VERSION=${VERSION} + +# Cloud Run +cloud_run: gcloud run deploy slo-generator \ - --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ - --region=${REGION} \ - --platform managed \ - --set-env-vars CONFIG_PATH=${CONFIG_URL} \ - --service-account=${SERVICE_ACCOUNT} \ - --project=${CLOUDRUN_PROJECT_ID} \ - --command="slo-generator" \ - --args=api \ - --args=--signature-type="${SIGNATURE_TYPE}" \ - --min-instances 1 \ - --allow-unauthenticated - -# Cloudrun - export mode only -cloudrun_export_only: + --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ + --region=${REGION} \ + --platform managed \ + --set-env-vars CONFIG_PATH=${CONFIG_URL} \ + --service-account=${SERVICE_ACCOUNT} \ + --project=${CLOUDRUN_PROJECT_ID} \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type="${SIGNATURE_TYPE}" \ + --min-instances 1 \ + --allow-unauthenticated + +# Cloud Run - Export Mode Only +cloud_run_export_only: gcloud run deploy slo-generator-export \ - --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ - --region=${REGION} \ - --platform managed \ - --set-env-vars CONFIG_PATH=${CONFIG_URL} \ - --service-account=${SERVICE_ACCOUNT} \ - --project=${CLOUDRUN_PROJECT_ID} \ - --command="slo-generator" \ - --args=api \ - --args=--signature-type="cloudevent" \ - --args=--target="run_export" \ - --min-instances 1 \ - --allow-unauthenticated - -gcloud_alpha: - gcloud components install alpha --quiet + --image gcr.io/${GCR_PROJECT_ID}/slo-generator:${VERSION} \ + --region=${REGION} \ + --platform managed \ + --set-env-vars CONFIG_PATH=${CONFIG_URL} \ + --service-account=${SERVICE_ACCOUNT} \ + --project=${CLOUDRUN_PROJECT_ID} \ + --command="slo-generator" \ + --args=api \ + --args=--signature-type="cloudevent" \ + --args=--target="run_export" \ + --min-instances 1 \ + --allow-unauthenticated diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 230d7e7f..c2532cb6 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,20 +13,15 @@ # limitations under the License. --- steps: - - name: gcr.io/cloud-builders/docker - id: Build SLO generator - args: ['build', '-t', 'gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION}', '.'] - -- name: gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION} - id: Run all tests - entrypoint: make - env: - - 'MIN_VALID_EVENTS=10' - args: [] + id: Build SLO Generator Docker Image + script: | + docker build \ + -t gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION} \ + . substitutions: _VERSION: latest images: -- 'gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION}' +- gcr.io/$_GCR_PROJECT_ID/slo-generator:${_VERSION} diff --git a/docs/deploy/cloudbuild.md b/docs/deploy/cloudbuild.md index 25988f57..d60a5e95 100644 --- a/docs/deploy/cloudbuild.md +++ b/docs/deploy/cloudbuild.md @@ -14,7 +14,7 @@ git clone https://github.com/google/slo-generator cd slo-generator/ export CLOUDBUILD_PROJECT_ID= export GCR_PROJECT_ID= -make cloudbuild +make cloud_build ``` ## Run `slo-generator` as a build step