Move dependencies needed in CI on prod images to separate requirement…

…s file.
mozilla · Nov 27, 2024 · 2003dc3 · 2003dc3
1 parent 83fa0f4
commit 2003dc3
Show file tree

Hide file tree

Showing 12 changed files with 543 additions and 354 deletions.
diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml
@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,8 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        # We can only build the production image in CI
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml
@@ -23,6 +23,10 @@ inputs:
     description: 'Skip data backup'
     required: false
     default: 'true'
+  install_ci_deps:
+    description: 'Install CI dependencies'
+    required: false
+    default: 'true'
 
 runs:
   using: 'composite'
@@ -40,6 +44,7 @@ runs:
         COMPOSE_FILE: ${{ inputs.compose_file }}
         HOST_UID: ${{ steps.id.outputs.id }}
         DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
+        INSTALL_CI_DEPS: ${{ inputs.install_ci_deps }}
       run: |
         # Start the specified services
         make up

diff --git a/.github/workflows/_test_make.yml b/.github/workflows/_test_make.yml
@@ -0,0 +1,67 @@
+name: Test Make up
+
+run-name: |
+  ref: ${{ github.ref_name }} |
+  version: ${{ inputs.version }} |
+  digest: ${{ inputs.digest }} |
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of the image to run
+        type: string
+        required: true
+      digest:
+        description: The build digest of the image to run. Overrides version.
+        type: string
+        required: false
+  workflow_dispatch:
+    inputs:
+      version:
+        description: The version of the image to run
+        type: string
+        required: true
+      digest:
+        description: The build digest of the image to run. Overrides version.
+        type: string
+        required: false
+
+concurrency:
+  group: test-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        docker_target:
+          - production
+          - development
+        docker_version:
+          - local
+          - ${{ inputs.version }}
+        install_ci_deps:
+          - true
+          - false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/run-docker
+        with:
+          version: ${{ matrix.docker_version }}
+          data_backup_skip: false
+          run: |
+            expected_deps=(prod)
+
+            if [ "${{ matrix.install_ci_deps }}" = "true" ]; then
+              expected_deps+=("ci")
+            fi
+
+            if [ "${{ matrix.docker_target }}" = "development" ]; then
+              expected_deps+=("dev")
+            fi
+
+            echo "Expected deps: ${expected_deps[@]}"
+            make check_
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   test_make_docker_configuration:
@@ -297,7 +296,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -329,5 +327,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
diff --git a/Dockerfile b/Dockerfile
@@ -86,26 +86,29 @@ ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-# Work arounds "Multiple .dist-info directories" issue.
-rm -rf /deps/build/*
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
+${HOME}/scripts/install_deps.py pip
 EOF
 
 # Expose the DOCKER_TARGET variable to all subsequent stages
 # This value is used to determine if we are building for production or development
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
 FROM base AS pip_production
 
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
@@ -115,27 +118,10 @@ RUN \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-npm ci ${NPM_ARGS} --include=prod
+${HOME}/scripts/install_deps.py prod
 EOF
 
-FROM base AS pip_development
-
-RUN \
-    # Files required to install pip dependencies
-    --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
-    --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
-    # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=${HOME}/package.json \
-    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
-    # Mounts for caching dependencies
-    --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-    --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
-EOF
+FROM base AS development
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -172,35 +158,23 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 
 ENV DOCKER_BUILD=${DOCKER_BUILD}
 ENV DOCKER_COMMIT=${DOCKER_COMMIT}
 ENV DOCKER_VERSION=${DOCKER_VERSION}
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
 
-
+# Set shell back to sh until we can prove we can use bash at runtime
+SHELL ["/bin/sh", "-c"]
diff --git a/Makefile-docker b/Makefile-docker
@@ -8,6 +8,9 @@ export PYTHON_COMMAND=python3
 export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
+# Get the docker version from the tag e.g repository/name:version@digest -> version
+override DOCKER_VERSION = $(shell echo $(DOCKER_TAG) | sed -n 's/.*:\([0-9]\.[0-9]\.[0-9]\).*/\1/p')
+
 NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
 
 REQUIRED_FILES := \
@@ -28,12 +31,15 @@ check_debian_packages: ## check the existence of multiple debian packages
 
 .PHONY: check_pip_packages
 check_pip_packages: ## check the existence of multiple python packages
-	@ ./scripts/check_pip_packages.sh prod.txt
-# "production" corresponds to the "propduction" DOCKER_TARGET defined in the Dockerfile
-# When the target is "production" it means we cannot expect dev.txt dependencies to be installed.
-	@if [ "$(DOCKER_TARGET)" != "production" ]; then \
-		./scripts/check_pip_packages.sh dev.txt; \
-	fi
+	./scripts/check_pip_packages.sh prod.txt
+# We expect dev dependencies only if DOCKER_TARGET is not production
+ifeq ($(DOCKER_TARGET), development)
+	./scripts/check_pip_packages.sh dev.txt
+endif
+# If we expect CI dependencies, then we should check for them
+ifneq ($(INSTALL_CI_DEPS),)
+	./scripts/check_pip_packages.sh ci.txt
+endif
 
 .PHONY: check_files
 check_files: ## check the existence of multiple files
@@ -79,6 +85,26 @@ update_assets:
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
 
+# Build list of dependencies to install
+DEPS = pip prod
+# If we expect CI dependencies, then we should install them
+ifdef INSTALL_CI_DEPS
+DEPS += ci
+endif
+# If we're running a development image, then we should install the development dependencies
+ifeq ($(DOCKER_TARGET), development)
+DEPS += dev
+endif
+
+.PHONY: update_deps
+update_deps: ## Update the dependencies
+# If we're running a non local or production image, remove the existing deps and install clean
+ifeq ($(or $(findstring local, $(DOCKER_TAG)), $(findstring development, $(DOCKER_TARGET))),)
+	rm -rf /deps
+endif
+# Install all dependencies in one call
+	$(HOME)/scripts/install_deps.py $(DEPS)
+
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests
 setup-ui-tests:

diff --git a/Makefile-os b/Makefile-os
@@ -11,6 +11,7 @@ export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
 export DATA_BACKUP_SKIP ?=
+export INSTALL_CI_DEPS ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
 INITIALIZE_ARGS ?=
@@ -43,7 +44,6 @@ DOCKER_COMPOSE_ARGS := \
 	--remove-orphans \
 	--no-build \
 	--quiet-pull \
-	--renew-anon-volumes
 
 # Paths should be cleaned before mounting .:/data/olympia
 # These are files which should be sourced from the container
@@ -54,7 +54,6 @@ CLEAN_PATHS := \
 	version.json \
 	logs \
 	buildx-bake-metadata.json \
-	deps \
 
 .PHONY: help_redirect
 help_redirect:
@@ -154,8 +153,22 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_image
 docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
 	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
 
+.PHONY: docker_compose_run
+docker_compose_run: ## Run a command in the docker compose project
+	docker compose run \
+		--rm \
+		--no-deps \
+		$(DOCKER_RUN_ARGS) \
+		web \
+		$(ARGS)
+
+.PHONY: docker_update_deps
+docker_update_deps: ## Update the dependencies in the container based on the docker tag and target
+# Install the production dependencies always since we mount the ./deps directory
+	$(MAKE) docker_compose_run ARGS='make update_deps'
+
 .PHONY: up
-up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
+up: setup docker_pull_or_build docker_update_deps docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
 	# Explicitly run initialize via the web container as make can get confused
 	# both routing the command to the web container and
 	# routing the command to the proper target.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -23,6 +23,8 @@ x-env-mapping: &env
     - HOST_UID
     - DEBUG
     - DATA_BACKUP_SKIP
+    - DOCKER_TARGET
+    - INSTALL_CI_DEPS
 
 x-olympia: &olympia
   <<: *env
@@ -52,6 +54,7 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
+      - ./deps:/deps
       - .:/data/olympia
       # Don't mount generated files. They only exist in the container
       # and would otherwiser be deleted by mounting the cwd volume above