Migrate dependencies to /data/olympia

Dockerfile

@@ -10,13 +10,24 @@ ENV BUILD_INFO=/build-info.json
 SHELL ["/bin/bash", "-xue", "-c"]
 
 ENV OLYMPIA_UID=9500
+# give olympia access to the HOME directory
+ENV HOME=/data/olympia
+ENV DEPS_DIR=${HOME}/deps
+ENV NPM_DEPS_DIR=${HOME}/node_modules
+
 RUN <<EOF
 groupadd -g ${OLYMPIA_UID} olympia
-useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d /data/olympia olympia
+useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d ${HOME} olympia
+
+# Create and chown olympia directories
+olympia_dirs=("${DEPS_DIR}" "${NPM_DEPS_DIR}" "${HOME}/storage")
+for dir in "${olympia_dirs[@]}"; do
+  mkdir -p ${dir}
+  chown -R olympia:olympia ${dir}
+done
 EOF
 
-# give olympia access to the HOME directory
-ENV HOME=/data/olympia
+
 WORKDIR ${HOME}
 RUN chown -R olympia:olympia ${HOME}
 
@@ -74,37 +85,28 @@ ENV LANG=en_US.UTF-8
 ENV LC_ALL=en_US.UTF-8
 
 RUN <<EOF
-# Create directory for dependencies
-mkdir /deps
-chown -R olympia:olympia /deps
-
-
 # For backwards-compatibility purposes, set up links to uwsgi. Note that
 # the target does not exist yet at this point, but it will later.
-ln -s /deps/bin/uwsgi /usr/bin/uwsgi
+ln -s ${DEPS_DIR}/bin/uwsgi /usr/bin/uwsgi
 ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
 
-# Create the storage directory and the test file to verify nginx routing
-mkdir -p ${HOME}/storage
-chown -R olympia:olympia ${HOME}/storage
 EOF
 
 USER olympia:olympia
 
 ENV PIP_USER=true
-ENV PIP_BUILD=/deps/build/
-ENV PIP_CACHE_DIR=/deps/cache/
-ENV PIP_SRC=/deps/src/
-ENV PYTHONUSERBASE=/deps
+ENV PIP_BUILD=${DEPS_DIR}/build/
+ENV PIP_CACHE_DIR=${DEPS_DIR}/cache/
+ENV PIP_SRC=${DEPS_DIR}/src/
+ENV PYTHONUSERBASE=${DEPS_DIR}
 ENV PATH=$PYTHONUSERBASE/bin:$PATH
-ENV NPM_CONFIG_PREFIX=/deps/
-ENV NPM_CACHE_DIR=/deps/cache/npm
+ENV NPM_CACHE_DIR=${DEPS_DIR}/cache/npm
 ENV NPM_DEBUG=true
 # Set python path to the project root and src to resolve olympia modules correctly
 ENV PYTHONPATH=${HOME}:${HOME}/src
 
 ENV PIP_COMMAND="python3 -m pip"
-ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel verbose"
+ENV NPM_ARGS="--cache ${NPM_CACHE_DIR} --loglevel verbose"
 
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
@@ -127,16 +129,16 @@ COPY docker/etc/mime.types /etc/mime.types
 
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
-# and makes copying the /deps dir to the final layer easy.
+# and makes copying the /data/olympia/deps dir to the final layer easy.
 FROM base AS pip_production
 
 RUN \
     --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=/deps/package.json \
-    --mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
+    --mount=type=bind,source=package.json,target=${HOME}/package.json \
+    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
@@ -146,6 +148,9 @@ EOF
 
 FROM base AS development
 
+# Copy build info from info
+COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -155,7 +160,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR}
 RUN \
     --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
     --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
+    --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \
     make -f Makefile-docker compile_locales
 
 # More efficient caching by mounting the exact files we need
@@ -175,10 +180,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
+make -f Makefile-docker update_assets
 EOF
 
 FROM base AS production
@@ -194,4 +199,4 @@ COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
-COPY --from=pip_production --chown=olympia:olympia /deps /deps
+COPY --from=pip_production --chown=olympia:olympia ${DEPS_DIR} ${DEPS_DIR}

Makefile-docker

@@ -8,16 +8,6 @@ export PYTHON_COMMAND=python3
 export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
-NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
-
-REQUIRED_FILES := \
-	Makefile \
-	Makefile-os \
-	Makefile-docker \
-	/deps/package.json \
-	/deps/package-lock.json \
-	/addons-server-docker-container \
-
 # Build list of dependencies to install
 DEPS = pip prod
 # If we're running a development image, then we should install the development dependencies
@@ -72,14 +62,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies
@@ -94,7 +78,7 @@ setup-ui-tests:
 lint: ## lint the code
 	ruff check .
 	ruff format --check .
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --check '**'
+	npm exec $(NPM_ARGS) -- prettier --check '**'
 	curlylint src/
 
 lint-codestyle: lint
@@ -199,15 +183,15 @@ test_failed: ## rerun the failed tests from the previous run
 
 .PHONY: run_js_tests
 run_js_tests: ## Run the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest tests/js
+	npm exec $(NPM_ARGS) -- jest tests/js
 
 .PHONY: watch_js_tests
 watch_js_tests: ## Run+watch the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest --watch
+	npm exec $(NPM_ARGS) -- jest --watch
 
 .PHONY: format
 format: ## Autoformat our codebase.
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --write '**'
+	npm exec $(NPM_ARGS) -- prettier --write '**'
 	ruff check --fix-only .
 	ruff format .
 
@@ -218,7 +202,7 @@ extract_locales: ## extracts and merges translation strings
 .PHONE: compile_locales
 compile_locales: ## compiles translation strings
 	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt
-	./locale/compile-mo.sh ./locale/
+	$(HOME)/scripts/compile_locales.py
 
 .PHONY: help_submake
 help_submake:

Makefile-os

@@ -153,17 +153,17 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
-.PHONY: docker_update_deps
-docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+.PHONY: docker_sync_host
+docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
 	docker compose run \
 		--rm \
 		--no-deps \
 		$(DOCKER_RUN_ARGS) \
 		web \
-		make update_deps
+		./scripts/sync_host_files.py
 
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers

docker-compose.yml

@@ -46,9 +46,9 @@ services:
     volumes:
       # used by: web, worker, nginx
       - ${HOST_MOUNT_SOURCE:?}:/data/olympia
-      - ${HOST_MOUNT_SOURCE:?}deps:/deps
-      - data_site_static:/data/olympia/site-static
-      - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage
+      - ${HOST_MOUNT_SOURCE:?}deps:/data/olympia/deps
+      - ./site-static:/data/olympia/site-static
+      - ./storage:/data/olympia/storage
   worker:
     <<: *olympia
     command: [
@@ -64,8 +64,8 @@ services:
     ]
     volumes:
       - ${HOST_MOUNT_SOURCE:?}:/data/olympia
-      - ${HOST_MOUNT_SOURCE:?}deps:/deps
-      - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage
+      - ${HOST_MOUNT_SOURCE:?}deps:/data/olympia/deps
+      - ./storage:/data/olympia/storage
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5

docker/entrypoint.sh

@@ -30,10 +30,10 @@ fi
 NEW_HOST_UID=$(get_olympia_uid)
 OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)"
 
-# If we are on production mode, update the ownership of /data/olympia and /deps to match the new id
+# If we are on production mode, update the ownership of /data/olympia to match the new id
 if [[ "${HOST_MOUNT}" == "production" ]]; then
-  echo "Updating ownership of /data/olympia and /deps to ${OLYMPIA_ID_STRING}"
-  chown -R ${OLYMPIA_ID_STRING} /data/olympia /deps
+  echo "Updating ownership of /data/olympia to ${OLYMPIA_ID_STRING}"
+  chown -R ${OLYMPIA_ID_STRING} /data/olympia
 fi
 
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER

docs/topics/development/building_and_running_services.md

@@ -14,7 +14,7 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
 
 3. **Mounts in Docker Compose**:
    - **Mounting Local Repository**: The volume `.:/data/olympia` mounts the local Git repository into the container, allowing real-time changes to files within the container.
-   - **Mounting Dependencies**: The volume `./deps:/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host.
+   - **Mounting Dependencies**: The volume `./deps:/data/olympia/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host.
 
 4. **Environment Variables for OLYMPIA_USER**:
    - **Development Setup**: The `OLYMPIA_UID` .env variable is set to the host user ID, ensuring that the container runs with the correct permissions.
@@ -178,4 +178,4 @@ Additionally this volume is "external" to allow the volume to persist across con
 We additionally mount serval local directories to the web/worker containers.
 
 - **.:/data/olympia**: Mounts the local repository into the container to allow real-time changes to files within the container.
-- **./deps:/deps**: Mounts the dependencies directory to enable better caching across builds and provide visibility for debugging directly on the host.
+- **./deps:/data/olympia/deps**: Mounts the dependencies directory to enable better caching across builds and provide visibility for debugging directly on the host.

docs/topics/development/dependency_management.md

@@ -4,9 +4,9 @@ Managing dependencies effectively is crucial for maintaining a stable and consis
 
 ## Python Dependencies
 
-Python dependencies are managed using the Makefile and requirements files. All dependencies are installed into the `/deps` directory, which centralizes dependency management and simplifies data mounts.
+Python dependencies are managed using the Makefile and requirements files. All dependencies are installed into the `/data/olympia/deps` directory, which centralizes dependency management and simplifies data mounts.
 
-- **Environment Variables**: The project sets environment variables for Python CLIs to install dependencies in specific locations. This includes setting paths for `PIP_CACHE_DIR`, `PIP_SRC`, and others to use the `/deps` directory.
+- **Environment Variables**: The project sets environment variables for Python CLIs to install dependencies in specific locations. This includes setting paths for `PIP_CACHE_DIR`, `PIP_SRC`, and others to use the `/data/olympia/deps` directory.
 
 - **Caching Mechanism**: By using Docker build stages, the project isolates the stages responsible for installing dependencies. This prevents these stages from re-running unless the actual dependency files are changed. Additionally, internal Python cache folders are cached, avoiding unnecessary re-downloads of packages and saving time and bandwidth.
 
@@ -66,9 +66,7 @@ Ensure to comment in the requirements file above transitive dependencies which d
 
 ## Node.js Dependencies
 
-Node.js dependencies are managed using npm. Similar to Python dependencies, Node.js dependencies are installed into the `/deps` directory.
-
-- **Environment Variables**: Environment variables are set for Node.js CLIs to ensure that dependencies are installed in the `/deps` directory. This includes setting paths for `NPM_CONFIG_PREFIX` and `NPM_CACHE_DIR`.
+Node.js dependencies are managed using npm.
 
 - **Caching Mechanism**: Node.js dependencies are also cached using Docker build stages. Internal npm cache folders are cached to avoid re-downloading packages unnecessarily.
 
@@ -86,19 +84,19 @@ NPM is a fully-featured package manager, so you can use the standard CLI.
 
 The Dockerfile uses build stages to isolate the dependency installation process. This ensures that stages do not re-run unless the dependency files themselves change. The caching mechanism includes:
 
-- **Dependency Cache**: Both Python and Node.js dependencies are cached in the `/deps` directory.
+- **Dependency Cache**: Both Python and Node.js dependencies are cached in the `/data/olympia/deps` directory.
 - **Cache Folders**: Internal cache folders for pip and npm are themselves cached to speed up the build process.
 
 ## GitHub Actions Cache
 
-The project uses a custom GitHub Actions action (`./.github/actions/cache-deps`) to cache the `/deps` folder. This action significantly increases install times for CI runs by leveraging the GitHub Actions cache.
+The project uses a custom GitHub Actions action (`./.github/actions/cache-deps`) to cache the `/data/olympia/deps` folder. This action significantly increases install times for CI runs by leveraging the GitHub Actions cache.
 
 ```yaml
 - name: Cache dependencies
   uses: ./.github/actions/cache-deps
 ```
 
-By caching the `/deps` folder, the project ensures that dependencies are quickly restored in CI environments, reducing overall build and test times.
+By caching the `/data/olympia/deps` folder, the project ensures that dependencies are quickly restored in CI environments, reducing overall build and test times.
 
 ## Updating/Installing Dependencies
 
@@ -109,4 +107,4 @@ make up
 ```
 
 This will rebuild the Docker image with the current dependencies specified in the `requirements` and `package.json` files.
-We do not support updating dependencies in a running container as the /deps folder is not writable by the olympia user.
+We do not support updating dependencies in a running container as the /data/olympia/deps folder is not writable by the olympia user.

docs/topics/development/performance_and_optimization.md

@@ -28,7 +28,7 @@ Docker layer caching is a powerful feature that significantly speeds up the buil
      export DOCKER_BUILDKIT=1
      ```
 
-   - **GitHub Actions Cache**: The custom action (`./.github/actions/cache-deps`) caches the `/deps` folder, leveraging GitHub Actions cache to improve CI run times.
+   - **GitHub Actions Cache**: The custom action (`./.github/actions/cache-deps`) caches the `/data/olympia/deps` folder, leveraging GitHub Actions cache to improve CI run times.
 
 ## Performance Testing
 

pyproject.toml

@@ -27,6 +27,8 @@ classifiers = [
 
 [tool.ruff]
 exclude = [
+    "deps",
+    "node_modules",
     "docs",
     "static",
     ".git",