build: Use buildpacks for better caching

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
     hooks:
       - id: insert-license
         name: Insert license headers (shell-style comments)
-        files: '(?:^|/)(?:.*\.(?:py|sh|toml|ya?ml|cfg|ini)|Dockerfile|Makefile|nginx.conf)$'
+        files: '(?:^|/)(?:.*\.(?:py|sh|toml|ya?ml|cfg|ini)|Dockerfile|Makefile|nginx.conf|build|detect|generate)$'
         exclude: '(?:^|/)\..+|^docs/Makefile$|^base/hooks/\+pre-commit.sh$'
         args:
           - --detect-license-in-X-top-lines=15

Makefile

@@ -112,15 +112,14 @@ INSTALL_OLD_GTK_VERSION ?= true
 PURE_VARIANTS_LICENSE_SERVER ?= http://localhost:8080
 PURE_VARIANTS_KNOWN_SERVERS ?= '[{"name": "test", "url": "http://example.localhost"}]'
 
-# Inject libraries from the capella/libs directory
-INJECT_LIBS_CAPELLA ?= false
-
 # Build architecture: amd64 or arm64
 BUILD_ARCHITECTURE ?= amd64
 
 DOCKER_BUILD_FLAGS ?= --platform linux/$(BUILD_ARCHITECTURE)
 DOCKER_RUN_FLAGS ?= --add-host=host.docker.internal:host-gateway --rm -it
-DOCKER_DEBUG_FLAGS ?= -it --entrypoint="bash" -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$$DISPLAY
+DOCKER_DEBUG_FLAGS ?= -it --entrypoint="bash"
+
+PACK_BUILD_FLAGS ?=
 
 # If set to 1, we will push the images to the specified registry
 PUSH_IMAGES ?= 0
@@ -169,6 +168,15 @@ all: \
 	t4c/client/remote/pure-variants \
 	capella/remote/pure-variants
 
+builder:
+	docker build buildpacks/builder \
+		--build-arg CNB_USER_ID=$(TECHUSER_UID) \
+		-t buildpacks-base
+	pack builder create $(DOCKER_REGISTRY)/mbse-builder \
+		--config buildpacks/builder/capella-builder.toml
+	docker push $(DOCKER_REGISTRY)/mbse-builder
+
+
 base: SHELL=/bin/bash
 base:
 	docker build $(DOCKER_BUILD_FLAGS) \
@@ -183,23 +191,23 @@ jupyter-notebook: base
 	$(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) DOCKER_TAG=$(DOCKER_TAG) IMAGENAME=$@ .push
 
 capella/base: SHELL=./capella_loop.sh
-capella/base: base
-	envsubst < capella/.dockerignore.template > capella/.dockerignore
-	cp eclipse/set_memory_flags.py capella/setup/set_memory_flags.py
-	docker build $(DOCKER_BUILD_FLAGS) \
-		-t $(DOCKER_PREFIX)$@:$$DOCKER_TAG \
-		--build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \
-		--build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$(CAPELLA_DOCKERIMAGES_REVISION) \
-		--build-arg BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \
-		--build-arg CAPELLA_VERSION=$$CAPELLA_VERSION \
-		--build-arg "CAPELLA_DROPINS=$(CAPELLA_DROPINS)" \
-		--build-arg "INJECT_PACKAGES=$(INJECT_LIBS_CAPELLA)" \
-		--build-arg INSTALL_OLD_GTK_VERSION=$(INSTALL_OLD_GTK_VERSION) \
-		capella
-	rm capella/.dockerignore
-	rm capella/setup/set_memory_flags.py
+capella/base: builder base
+	pack build $(DOCKER_PREFIX)$@:$$DOCKER_TAG \
+		--env CAPELLA_BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \
+		--env CAPELLA_VERSION=$$CAPELLA_VERSION \
+		--env CAPELLA_DROPINS="$(CAPELLA_DROPINS)" \
+		--uid $(TECHUSER_UID) \
+		--builder localhost:12345/mbse-builder \
+		--path ./buildpacks \
+		--pull-policy always \
+		--volume $$PWD:/app:ro \
+		--network host \
+		$(PACK_BUILD_FLAGS)
 	$(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) IMAGENAME=$@ .push
 
+# --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$(CAPELLA_DOCKERIMAGES_REVISION)
+# --build-arg INSTALL_OLD_GTK_VERSION=$(INSTALL_OLD_GTK_VERSION) \
+
 papyrus/base: DOCKER_TAG=$(PAPYRUS_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION)
 papyrus/base: DOCKER_BUILD_FLAGS=--platform linux/amd64
 papyrus/base: base
@@ -254,10 +262,19 @@ eclipse/remote/pure-variants: eclipse/remote
 	$(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) DOCKER_TAG=$(DOCKER_TAG) IMAGENAME=$@ .push
 
 t4c/client/base: SHELL=./capella_loop.sh
-t4c/client/base: capella/base
-	envsubst < t4c/.dockerignore.template > t4c/.dockerignore
-	docker build $(DOCKER_BUILD_FLAGS) -t $(DOCKER_PREFIX)$@:$$DOCKER_TAG --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$$DOCKER_TAG --build-arg CAPELLA_VERSION=$$CAPELLA_VERSION t4c
-	rm t4c/.dockerignore
+t4c/client/base: base
+	pack build $(DOCKER_PREFIX)$@:$$DOCKER_TAG \
+		--env CAPELLA_BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \
+		--env CAPELLA_VERSION=$$CAPELLA_VERSION \
+		--env CAPELLA_DROPINS="$(CAPELLA_DROPINS)" \
+		--env INSTALL_T4C_CLIENT=1 \
+		--uid $(TECHUSER_UID) \
+		--builder localhost:12345/mbse-builder \
+		--path ./buildpacks \
+		--pull-policy if-not-present \
+		--volume $$PWD:/app:ro \
+		--network host \
+		$(PACK_BUILD_FLAGS)
 	$(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) IMAGENAME=$@ .push
 
 t4c/client/remote: SHELL=./capella_loop.sh
@@ -312,7 +329,7 @@ run-capella/remote: capella/remote
 		-p $(RDP_PORT):3389 \
 		-p $(WEB_PORT):10000 \
 		-p $(METRICS_PORT):9118 \
-		$(DOCKER_PREFIX)$<:$$(echo "$(DOCKER_TAG_SCHEMA)" | envsubst)
+		$(DOCKER_PREFIX)capella/base:$$(echo "$(DOCKER_TAG_SCHEMA)" | envsubst)
 
 run-papyrus/remote: DOCKER_TAG=$(PAPYRUS_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION)
 run-papyrus/remote: papyrus/remote

base/Dockerfile

@@ -12,7 +12,7 @@ ENV SHELL=/bin/bash
 
 RUN apt-get update && \
     apt-get upgrade --yes && \
-    apt-get install --yes \
+    apt-get install --yes --no-install-recommends \
     gettext-base \
     locales \
     python3 \
@@ -46,7 +46,7 @@ ENV HOME=/home/techuser
 # This in analogous to the virtualenv activate script
 # To allow deactivation of the virtualenv, we need to save the old PATH
 ENV _OLD_VIRTUAL_PATH="$PATH"
-ENV VIRTUAL_ENV=/opt/.venv
+ENV VIRTUAL_ENV=/layers/venv/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 COPY --chmod=755 hooks/* /opt/git/global-hooks/
@@ -57,14 +57,15 @@ RUN ln -s "$(which python3.11)" /usr/bin/python && \
     ln -sf "$(which python3.11)" /usr/bin/python3 && \
     ln -sf "$(which pip3.11)" /usr/local/bin/pip && \
     ln -sf "$(which pip3.11)" /usr/local/bin/pip3 && \
-    python -m venv /opt/.venv && \
+    python -m venv /layers/venv/venv && \
     # Configure pre-commit
-    pip install --no-cache-dir pre-commit lxml PyYAML --no-cache-dir && \
+    pip install --no-cache-dir pre-commit lxml PyYAML && \
     echo "commit-msg post-rewrite pre-commit pre-merge-commit pre-rebase prepare-commit-msg" | xargs -n 1 cp /opt/git/global-hooks/+pre-commit-only.sh && \
     echo "pre-push post-checkout post-commit post-merge" | xargs -n 1 cp /opt/git/global-hooks/+pre-commit-and-lfs.sh && \
     git config --global core.hooksPath /opt/git/global-hooks && \
     chmod -R 755 /opt/git/global-hooks && \
-    chown -R techuser /opt/.venv/bin/ /opt/.venv/lib/python3.11/site-packages
+    chown -R techuser ${VIRTUAL_ENV}/bin/ ${VIRTUAL_ENV}/lib/python3.11/site-packages && \
+    chown -R techuser /var/log
 
 # Make pre-commit cache persistent
 ENV PRE_COMMIT_HOME=/workspace/.pre-commit

builder/Dockerfile

@@ -3,7 +3,6 @@
 
 FROM python:3.11-bookworm
 
-
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 ENV SHELL=/bin/bash
 

buildpacks/builder/Dockerfile

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Define the base image
+ARG BASE_IMAGE=debian:bookworm-slim
+FROM $BASE_IMAGE
+
+USER root
+
+# Install packages that we want to make available at build time
+RUN apt-get update && \
+  apt-get install -y \
+    xz-utils \
+    ca-certificates \
+    rsync \
+    python3 \
+    python3-pip \
+    python3-venv && \
+  rm -rf /var/lib/apt/lists/*
+
+# Set required CNB user information
+ARG CNB_USER_ID=1000
+ARG CNB_GROUP_ID=1000
+ENV CNB_USER_ID=${CNB_USER_ID}
+ENV CNB_GROUP_ID=${CNB_GROUP_ID}
+
+# Create user and group
+RUN groupadd cnb --gid ${CNB_GROUP_ID} && \
+  useradd -l --uid ${CNB_USER_ID} --gid ${CNB_GROUP_ID} -m -s /bin/bash cnb
+
+RUN ln -s "$(which python3.11)" /usr/bin/python && \
+  ln -sf "$(which python3.11)" /usr/bin/python3 && \
+  ln -sf "$(which pip3.11)" /usr/local/bin/pip && \
+  ln -sf "$(which pip3.11)" /usr/local/bin/pip3 && \
+  pip install --break-system-packages --no-cache-dir pre-commit lxml PyYAML
+
+# Set user and group
+USER ${CNB_USER_ID}:${CNB_GROUP_ID}

buildpacks/builder/capella-builder.toml

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+[[buildpacks]]
+uri = "../buildpacks/capella"
+
+[[buildpacks]]
+uri = "../buildpacks/capella-dropins"
+
+[[buildpacks]]
+uri = "../buildpacks/t4c"
+
+[[buildpacks]]
+uri = "../buildpacks/supervisord"
+
+[[buildpacks]]
+uri = "../buildpacks/xpra"
+
+[[buildpacks]]
+uri = "../buildpacks/xrdp"
+
+[[buildpacks]]
+uri = "../buildpacks/xidletime"
+
+[[extensions]]
+uri = "../extensions/capella-deps"
+
+[[extensions]]
+uri = "../extensions/xpra-deps"
+
+[[extensions]]
+
+uri = "../extensions/xrdp-deps"
+
+[[extensions]]
+uri = "../extensions/xidletime-deps"
+
+[[order]]
+group = [
+    { id = "capella", version = "0.0.0" },
+    { id = "capella-dropins", version = "0.0.0" },
+    { id = "t4c", version = "0.0.0", optional = true },
+    { id = "supervisord", version = "0.0.0", optional = true },
+    { id = "xpra", version = "0.0.0", optional = true },
+    { id = "xrdp", version = "0.0.0", optional = true },
+    { id = "xidletime", version = "0.0.0", optional = true },
+]
+
+[[order-extensions]]
+group = [
+    { id = "capella-deps", version = "0.0.0" },
+    { id = "xpra-deps", version = "0.0.0", optional = true },
+    { id = "xrdp-deps", version = "0.0.0", optional = true },
+    { id = "xidletime-deps", version = "0.0.0", optional = true },
+]
+
+[build]
+image = "buildpacks-base"
+[run]
+[[run.images]]
+image = "base"
+mirrors = ["localhost:12345/base"]

buildpacks/buildpacks/capella-dropins/bin/build

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -eo pipefail
+
+mkdir -p ${CNB_LAYERS_DIR}/app
+cp -r /app/capella/versions/${CAPELLA_VERSION}/dropins/* ${CNB_LAYERS_DIR}/app/
+
+# In case someone had dropins in their archive, copy it to the dropins layer
+if [ -n "$(ls -A /layers/capella/app/dropins 2>/dev/null)" ]; then
+    cp -r "/layers/capella/app/dropins/"* ${CNB_LAYERS_DIR}/app/
+fi
+
+# Replace dropins folder with symbolic link to dropins layer
+rm -rf /layers/capella/app/dropins
+ln -s ${CNB_LAYERS_DIR}/app /layers/capella/app/dropins
+
+python3 /app/buildpacks/buildpacks/capella-dropins/install_dropins.py
+
+cat > "${CNB_LAYERS_DIR}/app.toml" << EOL
+[types]
+launch = true
+EOL

buildpacks/buildpacks/capella-dropins/bin/detect

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -eo pipefail
+
+# Don't cache if capella layer has changed. The symlink has to be updated!
+
+# Otherwise cache if hash of dropins has not changed.

buildpacks/buildpacks/capella-dropins/buildpack.toml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+api = "0.11"
+
+[buildpack]
+id = "capella-dropins"
+version = "0.0.0"
+
+[[targets]]
+os = "linux"

capella/install_dropins.py → buildpacks/buildpacks/capella-dropins/install_dropins.py

@@ -9,10 +9,14 @@
 
 import yaml
 
+CAPELLA_VERSION = os.environ["CAPELLA_VERSION"]
+
 
 def load_dropins() -> dict[str, t.Any]:
     return yaml.safe_load(
-        pathlib.Path("/opt/dropins.yml").read_text(encoding="utf-8")
+        pathlib.Path(
+            "/app/capella/versions", CAPELLA_VERSION, "dropins.yml"
+        ).read_text(encoding="utf-8")
     )["dropins"]
 
 
@@ -44,7 +48,7 @@ def extract_repositories_and_install_ius(dropins: dict[str, t.Any]) -> None:
 def install_update_sites(repository: str, install_ui: list[str]) -> None:
     subprocess.run(
         [
-            "/opt/capella/capella",
+            "/layers/capella/app/capella",
             "-consoleLog",
             "-application",
             "org.eclipse.equinox.p2.director",
@@ -60,7 +64,8 @@ def install_update_sites(repository: str, install_ui: list[str]) -> None:
 
 def download_and_copy_dropin(download_url: str, file_name: str) -> None:
     urllib.request.urlretrieve(
-        download_url, pathlib.Path("/opt/capella/dropins") / file_name
+        download_url,
+        pathlib.Path("/layers/capella/app/dropins") / file_name,
     )
 
 

buildpacks/buildpacks/capella/autostart

@@ -0,0 +1,17 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Autostart script
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+if [ "$AUTOSTART_CAPELLA" = "1" ];
+then
+    if [ "$RESTART_CAPELLA" = "1" ];
+    then
+        # Run capella in a loop:
+        ( while true; do /layers/capella/app/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log; sleep 1; done ) &
+    else
+        /layers/capella/app/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log &
+    fi
+fi