From ae6553bf870c0b79189cc54d57ee5db0a961df1c Mon Sep 17 00:00:00 2001 From: MoritzWeber Date: Mon, 1 Jul 2024 21:14:14 +0200 Subject: [PATCH] build: Use buildpacks for better caching --- .pre-commit-config.yaml | 2 +- Makefile | 65 ++++--- base/Dockerfile | 11 +- builder/Dockerfile | 1 - buildpacks/builder/Dockerfile | 38 ++++ buildpacks/builder/capella-builder.toml | 62 +++++++ .../buildpacks/capella-dropins/bin/build | 24 +++ .../buildpacks/capella-dropins/bin/detect | 9 + .../buildpacks/capella-dropins/buildpack.toml | 11 ++ .../capella-dropins}/install_dropins.py | 11 +- buildpacks/buildpacks/capella/autostart | 17 ++ buildpacks/buildpacks/capella/bin/build | 78 ++++++++ buildpacks/buildpacks/capella/bin/detect | 47 +++++ buildpacks/buildpacks/capella/buildpack.toml | 11 ++ .../buildpacks/capella}/download_archive.py | 2 +- .../buildpacks/capella/entrypoint.sh | 6 +- .../capella/exec.d}/provisioning.py | 6 +- .../capella/exec.d/set_autostart.sh | 8 + .../capella/exec.d}/setup_workspace.py | 2 + buildpacks/buildpacks/supervisord/bin/build | 28 +++ buildpacks/buildpacks/supervisord/bin/detect | 5 + .../buildpacks/supervisord/buildpack.toml | 12 ++ buildpacks/buildpacks/t4c/bin/build | 63 +++++++ buildpacks/buildpacks/t4c/bin/detect | 15 ++ buildpacks/buildpacks/t4c/buildpack.toml | 11 ++ .../buildpacks/t4c/entrypoint.sh | 6 +- .../t4c/exec.d/inject_in_capella_ini.sh | 11 ++ .../t4c/exec.d/reconstruct_layer.sh | 7 + .../t4c/exec.d}/setup_workspace_t4c.py | 0 buildpacks/buildpacks/xidletime/bin/build | 25 +++ buildpacks/buildpacks/xidletime/bin/detect | 5 + .../buildpacks/xidletime/buildpack.toml | 12 ++ .../buildpacks/xidletime}/metrics.py | 0 buildpacks/buildpacks/xpra/bin/build | 18 ++ buildpacks/buildpacks/xpra/bin/detect | 5 + buildpacks/buildpacks/xpra/buildpack.toml | 12 ++ .../buildpacks/xpra}/error.html | 0 .../xpra/exec.d/add_supervisord_config.sh | 8 +- .../xpra/exec.d/replace_nginx_vars.sh | 8 + .../buildpacks/xpra}/nginx.conf | 2 +- buildpacks/buildpacks/xrdp/bin/build | 18 ++ buildpacks/buildpacks/xrdp/bin/detect | 5 + buildpacks/buildpacks/xrdp/buildpack.toml | 12 ++ .../xrdp/exec.d/add_supervisord_config.sh | 7 + .../xrdp/exec.d/copy_openbox_config.sh | 9 + .../buildpacks/xrdp/exec.d/set_password.sh | 20 ++ .../buildpacks/xrdp}/menu.xml | 0 {remote => buildpacks/buildpacks/xrdp}/rc.xml | 0 buildpacks/extensions/capella-deps/bin/detect | 5 + .../extensions/capella-deps/bin/generate | 37 ++++ .../extensions/capella-deps/extension.toml | 9 + .../extensions/xidletime-deps/bin/detect | 5 + .../extensions/xidletime-deps/bin/generate | 21 +++ .../extensions/xidletime-deps/extension.toml | 9 + buildpacks/extensions/xpra-deps/bin/detect | 5 + buildpacks/extensions/xpra-deps/bin/generate | 38 ++++ .../extensions/xpra-deps/extension.toml | 9 + buildpacks/extensions/xrdp-deps/bin/detect | 5 + buildpacks/extensions/xrdp-deps/bin/generate | 32 ++++ .../extensions/xrdp-deps/extension.toml | 9 + capella/.dockerignore.template | 7 - capella/.dockerignore.template.license | 2 - capella/Dockerfile | 174 ------------------ capella/autostart | 22 --- capella/libs/.gitkeep | 0 capella/patch.sh | 6 +- capella/setup/README.md | 9 - capella/setup/replace_env_variables.sh | 7 - capella/startup.sh | 8 - ci-templates/gitlab/diagram-cache.yml | 22 +-- ci-templates/gitlab/model-validation.yml | 30 +-- docs/docs/design-decisions/buildpacks.md | 61 ++++++ docs/mkdocs.yml | 2 + eclipse/set_memory_flags.py | 6 +- remote/Dockerfile | 90 --------- remote/startup.sh | 55 ------ remote/supervisord.conf | 13 -- remote/tests/test_metrics.py | 68 ------- remote/wallpaper.png | 0 t4c/.dockerignore.template | 7 - t4c/.dockerignore.template.license | 2 - t4c/Dockerfile | 70 ------- t4c/t4c_cli/util/capella.py | 2 +- 83 files changed, 970 insertions(+), 612 deletions(-) create mode 100644 buildpacks/builder/Dockerfile create mode 100644 buildpacks/builder/capella-builder.toml create mode 100755 buildpacks/buildpacks/capella-dropins/bin/build create mode 100755 buildpacks/buildpacks/capella-dropins/bin/detect create mode 100644 buildpacks/buildpacks/capella-dropins/buildpack.toml rename {capella => buildpacks/buildpacks/capella-dropins}/install_dropins.py (84%) create mode 100755 buildpacks/buildpacks/capella/autostart create mode 100755 buildpacks/buildpacks/capella/bin/build create mode 100755 buildpacks/buildpacks/capella/bin/detect create mode 100644 buildpacks/buildpacks/capella/buildpack.toml rename {capella => buildpacks/buildpacks/capella}/download_archive.py (96%) rename remote/bg-saved.cfg => buildpacks/buildpacks/capella/entrypoint.sh (65%) mode change 100644 => 100755 rename {capella/setup => buildpacks/buildpacks/capella/exec.d}/provisioning.py (98%) mode change 100644 => 100755 create mode 100755 buildpacks/buildpacks/capella/exec.d/set_autostart.sh rename {capella/setup => buildpacks/buildpacks/capella/exec.d}/setup_workspace.py (99%) mode change 100644 => 100755 create mode 100755 buildpacks/buildpacks/supervisord/bin/build create mode 100755 buildpacks/buildpacks/supervisord/bin/detect create mode 100644 buildpacks/buildpacks/supervisord/buildpack.toml create mode 100755 buildpacks/buildpacks/t4c/bin/build create mode 100755 buildpacks/buildpacks/t4c/bin/detect create mode 100644 buildpacks/buildpacks/t4c/buildpack.toml rename t4c/docker_entrypoint.sh => buildpacks/buildpacks/t4c/entrypoint.sh (69%) create mode 100755 buildpacks/buildpacks/t4c/exec.d/inject_in_capella_ini.sh create mode 100755 buildpacks/buildpacks/t4c/exec.d/reconstruct_layer.sh rename {t4c => buildpacks/buildpacks/t4c/exec.d}/setup_workspace_t4c.py (100%) create mode 100755 buildpacks/buildpacks/xidletime/bin/build create mode 100755 buildpacks/buildpacks/xidletime/bin/detect create mode 100644 buildpacks/buildpacks/xidletime/buildpack.toml rename {remote => buildpacks/buildpacks/xidletime}/metrics.py (100%) create mode 100755 buildpacks/buildpacks/xpra/bin/build create mode 100755 buildpacks/buildpacks/xpra/bin/detect create mode 100644 buildpacks/buildpacks/xpra/buildpack.toml rename {remote => buildpacks/buildpacks/xpra}/error.html (100%) rename remote/supervisord.xpra.conf => buildpacks/buildpacks/xpra/exec.d/add_supervisord_config.sh (72%) mode change 100644 => 100755 create mode 100755 buildpacks/buildpacks/xpra/exec.d/replace_nginx_vars.sh rename {remote => buildpacks/buildpacks/xpra}/nginx.conf (96%) create mode 100755 buildpacks/buildpacks/xrdp/bin/build create mode 100755 buildpacks/buildpacks/xrdp/bin/detect create mode 100644 buildpacks/buildpacks/xrdp/buildpack.toml rename remote/supervisord.xrdp.conf => buildpacks/buildpacks/xrdp/exec.d/add_supervisord_config.sh (66%) mode change 100644 => 100755 create mode 100755 buildpacks/buildpacks/xrdp/exec.d/copy_openbox_config.sh create mode 100755 buildpacks/buildpacks/xrdp/exec.d/set_password.sh rename {remote => buildpacks/buildpacks/xrdp}/menu.xml (100%) rename {remote => buildpacks/buildpacks/xrdp}/rc.xml (100%) create mode 100755 buildpacks/extensions/capella-deps/bin/detect create mode 100755 buildpacks/extensions/capella-deps/bin/generate create mode 100644 buildpacks/extensions/capella-deps/extension.toml create mode 100755 buildpacks/extensions/xidletime-deps/bin/detect create mode 100755 buildpacks/extensions/xidletime-deps/bin/generate create mode 100644 buildpacks/extensions/xidletime-deps/extension.toml create mode 100755 buildpacks/extensions/xpra-deps/bin/detect create mode 100755 buildpacks/extensions/xpra-deps/bin/generate create mode 100644 buildpacks/extensions/xpra-deps/extension.toml create mode 100755 buildpacks/extensions/xrdp-deps/bin/detect create mode 100755 buildpacks/extensions/xrdp-deps/bin/generate create mode 100644 buildpacks/extensions/xrdp-deps/extension.toml delete mode 100644 capella/.dockerignore.template delete mode 100644 capella/.dockerignore.template.license delete mode 100644 capella/Dockerfile delete mode 100755 capella/autostart delete mode 100644 capella/libs/.gitkeep delete mode 100644 capella/setup/README.md delete mode 100755 capella/setup/replace_env_variables.sh delete mode 100755 capella/startup.sh create mode 100644 docs/docs/design-decisions/buildpacks.md mode change 100644 => 100755 eclipse/set_memory_flags.py delete mode 100644 remote/Dockerfile delete mode 100755 remote/startup.sh delete mode 100644 remote/supervisord.conf delete mode 100644 remote/tests/test_metrics.py delete mode 100644 remote/wallpaper.png delete mode 100644 t4c/.dockerignore.template delete mode 100644 t4c/.dockerignore.template.license delete mode 100644 t4c/Dockerfile diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 866d8537..2f81183d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: hooks: - id: insert-license name: Insert license headers (shell-style comments) - files: '(?:^|/)(?:.*\.(?:py|sh|toml|ya?ml|cfg|ini)|Dockerfile|Makefile|nginx.conf)$' + files: '(?:^|/)(?:.*\.(?:py|sh|toml|ya?ml|cfg|ini)|Dockerfile|Makefile|nginx.conf|build|detect|generate)$' exclude: '(?:^|/)\..+|^docs/Makefile$|^base/hooks/\+pre-commit.sh$' args: - --detect-license-in-X-top-lines=15 diff --git a/Makefile b/Makefile index f9f4ab30..f72bb5ed 100644 --- a/Makefile +++ b/Makefile @@ -112,15 +112,14 @@ INSTALL_OLD_GTK_VERSION ?= true PURE_VARIANTS_LICENSE_SERVER ?= http://localhost:8080 PURE_VARIANTS_KNOWN_SERVERS ?= '[{"name": "test", "url": "http://example.localhost"}]' -# Inject libraries from the capella/libs directory -INJECT_LIBS_CAPELLA ?= false - # Build architecture: amd64 or arm64 BUILD_ARCHITECTURE ?= amd64 DOCKER_BUILD_FLAGS ?= --platform linux/$(BUILD_ARCHITECTURE) DOCKER_RUN_FLAGS ?= --add-host=host.docker.internal:host-gateway --rm -it -DOCKER_DEBUG_FLAGS ?= -it --entrypoint="bash" -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$$DISPLAY +DOCKER_DEBUG_FLAGS ?= -it --entrypoint="bash" + +PACK_BUILD_FLAGS ?= # If set to 1, we will push the images to the specified registry PUSH_IMAGES ?= 0 @@ -169,6 +168,15 @@ all: \ t4c/client/remote/pure-variants \ capella/remote/pure-variants +builder: + docker build buildpacks/builder \ + --build-arg CNB_USER_ID=$(TECHUSER_UID) \ + -t buildpacks-base + pack builder create $(DOCKER_REGISTRY)/mbse-builder \ + --config buildpacks/builder/capella-builder.toml + docker push $(DOCKER_REGISTRY)/mbse-builder + + base: SHELL=/bin/bash base: docker build $(DOCKER_BUILD_FLAGS) \ @@ -183,23 +191,23 @@ jupyter-notebook: base $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) DOCKER_TAG=$(DOCKER_TAG) IMAGENAME=$@ .push capella/base: SHELL=./capella_loop.sh -capella/base: base - envsubst < capella/.dockerignore.template > capella/.dockerignore - cp eclipse/set_memory_flags.py capella/setup/set_memory_flags.py - docker build $(DOCKER_BUILD_FLAGS) \ - -t $(DOCKER_PREFIX)$@:$$DOCKER_TAG \ - --build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \ - --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$(CAPELLA_DOCKERIMAGES_REVISION) \ - --build-arg BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \ - --build-arg CAPELLA_VERSION=$$CAPELLA_VERSION \ - --build-arg "CAPELLA_DROPINS=$(CAPELLA_DROPINS)" \ - --build-arg "INJECT_PACKAGES=$(INJECT_LIBS_CAPELLA)" \ - --build-arg INSTALL_OLD_GTK_VERSION=$(INSTALL_OLD_GTK_VERSION) \ - capella - rm capella/.dockerignore - rm capella/setup/set_memory_flags.py +capella/base: builder base + pack build $(DOCKER_PREFIX)$@:$$DOCKER_TAG \ + --env CAPELLA_BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \ + --env CAPELLA_VERSION=$$CAPELLA_VERSION \ + --env CAPELLA_DROPINS="$(CAPELLA_DROPINS)" \ + --uid $(TECHUSER_UID) \ + --builder localhost:12345/mbse-builder \ + --path ./buildpacks \ + --pull-policy always \ + --volume $$PWD:/app:ro \ + --network host \ + $(PACK_BUILD_FLAGS) $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) IMAGENAME=$@ .push +# --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$(CAPELLA_DOCKERIMAGES_REVISION) +# --build-arg INSTALL_OLD_GTK_VERSION=$(INSTALL_OLD_GTK_VERSION) \ + papyrus/base: DOCKER_TAG=$(PAPYRUS_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION) papyrus/base: DOCKER_BUILD_FLAGS=--platform linux/amd64 papyrus/base: base @@ -254,10 +262,19 @@ eclipse/remote/pure-variants: eclipse/remote $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) DOCKER_TAG=$(DOCKER_TAG) IMAGENAME=$@ .push t4c/client/base: SHELL=./capella_loop.sh -t4c/client/base: capella/base - envsubst < t4c/.dockerignore.template > t4c/.dockerignore - docker build $(DOCKER_BUILD_FLAGS) -t $(DOCKER_PREFIX)$@:$$DOCKER_TAG --build-arg BASE_IMAGE=$(DOCKER_PREFIX)$<:$$DOCKER_TAG --build-arg CAPELLA_VERSION=$$CAPELLA_VERSION t4c - rm t4c/.dockerignore +t4c/client/base: base + pack build $(DOCKER_PREFIX)$@:$$DOCKER_TAG \ + --env CAPELLA_BUILD_TYPE=$(CAPELLA_BUILD_TYPE) \ + --env CAPELLA_VERSION=$$CAPELLA_VERSION \ + --env CAPELLA_DROPINS="$(CAPELLA_DROPINS)" \ + --env INSTALL_T4C_CLIENT=1 \ + --uid $(TECHUSER_UID) \ + --builder localhost:12345/mbse-builder \ + --path ./buildpacks \ + --pull-policy if-not-present \ + --volume $$PWD:/app:ro \ + --network host \ + $(PACK_BUILD_FLAGS) $(MAKE) PUSH_IMAGES=$(PUSH_IMAGES) IMAGENAME=$@ .push t4c/client/remote: SHELL=./capella_loop.sh @@ -312,7 +329,7 @@ run-capella/remote: capella/remote -p $(RDP_PORT):3389 \ -p $(WEB_PORT):10000 \ -p $(METRICS_PORT):9118 \ - $(DOCKER_PREFIX)$<:$$(echo "$(DOCKER_TAG_SCHEMA)" | envsubst) + $(DOCKER_PREFIX)capella/base:$$(echo "$(DOCKER_TAG_SCHEMA)" | envsubst) run-papyrus/remote: DOCKER_TAG=$(PAPYRUS_VERSION)-$(CAPELLA_DOCKERIMAGES_REVISION) run-papyrus/remote: papyrus/remote diff --git a/base/Dockerfile b/base/Dockerfile index 752f6fd8..1c470801 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -12,7 +12,7 @@ ENV SHELL=/bin/bash RUN apt-get update && \ apt-get upgrade --yes && \ - apt-get install --yes \ + apt-get install --yes --no-install-recommends \ gettext-base \ locales \ python3 \ @@ -46,7 +46,7 @@ ENV HOME=/home/techuser # This in analogous to the virtualenv activate script # To allow deactivation of the virtualenv, we need to save the old PATH ENV _OLD_VIRTUAL_PATH="$PATH" -ENV VIRTUAL_ENV=/opt/.venv +ENV VIRTUAL_ENV=/layers/venv/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY --chmod=755 hooks/* /opt/git/global-hooks/ @@ -57,14 +57,15 @@ RUN ln -s "$(which python3.11)" /usr/bin/python && \ ln -sf "$(which python3.11)" /usr/bin/python3 && \ ln -sf "$(which pip3.11)" /usr/local/bin/pip && \ ln -sf "$(which pip3.11)" /usr/local/bin/pip3 && \ - python -m venv /opt/.venv && \ + python -m venv /layers/venv/venv && \ # Configure pre-commit - pip install --no-cache-dir pre-commit lxml PyYAML --no-cache-dir && \ + pip install --no-cache-dir pre-commit lxml PyYAML && \ echo "commit-msg post-rewrite pre-commit pre-merge-commit pre-rebase prepare-commit-msg" | xargs -n 1 cp /opt/git/global-hooks/+pre-commit-only.sh && \ echo "pre-push post-checkout post-commit post-merge" | xargs -n 1 cp /opt/git/global-hooks/+pre-commit-and-lfs.sh && \ git config --global core.hooksPath /opt/git/global-hooks && \ chmod -R 755 /opt/git/global-hooks && \ - chown -R techuser /opt/.venv/bin/ /opt/.venv/lib/python3.11/site-packages + chown -R techuser ${VIRTUAL_ENV}/bin/ ${VIRTUAL_ENV}/lib/python3.11/site-packages && \ + chown -R techuser /var/log # Make pre-commit cache persistent ENV PRE_COMMIT_HOME=/workspace/.pre-commit diff --git a/builder/Dockerfile b/builder/Dockerfile index 9908a4ee..f6755492 100644 --- a/builder/Dockerfile +++ b/builder/Dockerfile @@ -3,7 +3,6 @@ FROM python:3.11-bookworm - SHELL ["/bin/bash", "-euo", "pipefail", "-c"] ENV SHELL=/bin/bash diff --git a/buildpacks/builder/Dockerfile b/buildpacks/builder/Dockerfile new file mode 100644 index 00000000..6b8f0674 --- /dev/null +++ b/buildpacks/builder/Dockerfile @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +# Define the base image +ARG BASE_IMAGE=debian:bookworm-slim +FROM $BASE_IMAGE + +USER root + +# Install packages that we want to make available at build time +RUN apt-get update && \ + apt-get install -y \ + xz-utils \ + ca-certificates \ + rsync \ + python3 \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +# Set required CNB user information +ARG CNB_USER_ID=1000 +ARG CNB_GROUP_ID=1000 +ENV CNB_USER_ID=${CNB_USER_ID} +ENV CNB_GROUP_ID=${CNB_GROUP_ID} + +# Create user and group +RUN groupadd cnb --gid ${CNB_GROUP_ID} && \ + useradd -l --uid ${CNB_USER_ID} --gid ${CNB_GROUP_ID} -m -s /bin/bash cnb + +RUN ln -s "$(which python3.11)" /usr/bin/python && \ + ln -sf "$(which python3.11)" /usr/bin/python3 && \ + ln -sf "$(which pip3.11)" /usr/local/bin/pip && \ + ln -sf "$(which pip3.11)" /usr/local/bin/pip3 && \ + pip install --break-system-packages --no-cache-dir pre-commit lxml PyYAML + +# Set user and group +USER ${CNB_USER_ID}:${CNB_GROUP_ID} diff --git a/buildpacks/builder/capella-builder.toml b/buildpacks/builder/capella-builder.toml new file mode 100644 index 00000000..d6dbd3c5 --- /dev/null +++ b/buildpacks/builder/capella-builder.toml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +[[buildpacks]] +uri = "../buildpacks/capella" + +[[buildpacks]] +uri = "../buildpacks/capella-dropins" + +[[buildpacks]] +uri = "../buildpacks/t4c" + +[[buildpacks]] +uri = "../buildpacks/supervisord" + +[[buildpacks]] +uri = "../buildpacks/xpra" + +[[buildpacks]] +uri = "../buildpacks/xrdp" + +[[buildpacks]] +uri = "../buildpacks/xidletime" + +[[extensions]] +uri = "../extensions/capella-deps" + +[[extensions]] +uri = "../extensions/xpra-deps" + +[[extensions]] + +uri = "../extensions/xrdp-deps" + +[[extensions]] +uri = "../extensions/xidletime-deps" + +[[order]] +group = [ + { id = "capella", version = "0.0.0" }, + { id = "capella-dropins", version = "0.0.0" }, + { id = "t4c", version = "0.0.0", optional = true }, + { id = "supervisord", version = "0.0.0", optional = true }, + { id = "xpra", version = "0.0.0", optional = true }, + { id = "xrdp", version = "0.0.0", optional = true }, + { id = "xidletime", version = "0.0.0", optional = true }, +] + +[[order-extensions]] +group = [ + { id = "capella-deps", version = "0.0.0" }, + { id = "xpra-deps", version = "0.0.0", optional = true }, + { id = "xrdp-deps", version = "0.0.0", optional = true }, + { id = "xidletime-deps", version = "0.0.0", optional = true }, +] + +[build] +image = "buildpacks-base" +[run] +[[run.images]] +image = "base" +mirrors = ["localhost:12345/base"] diff --git a/buildpacks/buildpacks/capella-dropins/bin/build b/buildpacks/buildpacks/capella-dropins/bin/build new file mode 100755 index 00000000..62475df6 --- /dev/null +++ b/buildpacks/buildpacks/capella-dropins/bin/build @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p ${CNB_LAYERS_DIR}/app +cp -r /app/capella/versions/${CAPELLA_VERSION}/dropins/* ${CNB_LAYERS_DIR}/app/ + +# In case someone had dropins in their archive, copy it to the dropins layer +if [ -n "$(ls -A /layers/capella/app/dropins 2>/dev/null)" ]; then + cp -r "/layers/capella/app/dropins/"* ${CNB_LAYERS_DIR}/app/ +fi + +# Replace dropins folder with symbolic link to dropins layer +rm -rf /layers/capella/app/dropins +ln -s ${CNB_LAYERS_DIR}/app /layers/capella/app/dropins + +python3 /app/buildpacks/buildpacks/capella-dropins/install_dropins.py + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL diff --git a/buildpacks/buildpacks/capella-dropins/bin/detect b/buildpacks/buildpacks/capella-dropins/bin/detect new file mode 100755 index 00000000..93686406 --- /dev/null +++ b/buildpacks/buildpacks/capella-dropins/bin/detect @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +# Don't cache if capella layer has changed. The symlink has to be updated! + +# Otherwise cache if hash of dropins has not changed. diff --git a/buildpacks/buildpacks/capella-dropins/buildpack.toml b/buildpacks/buildpacks/capella-dropins/buildpack.toml new file mode 100644 index 00000000..d27f39f9 --- /dev/null +++ b/buildpacks/buildpacks/capella-dropins/buildpack.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "capella-dropins" +version = "0.0.0" + +[[targets]] +os = "linux" diff --git a/capella/install_dropins.py b/buildpacks/buildpacks/capella-dropins/install_dropins.py similarity index 84% rename from capella/install_dropins.py rename to buildpacks/buildpacks/capella-dropins/install_dropins.py index 0aef7ef5..f31a3e3e 100644 --- a/capella/install_dropins.py +++ b/buildpacks/buildpacks/capella-dropins/install_dropins.py @@ -9,10 +9,14 @@ import yaml +CAPELLA_VERSION = os.environ["CAPELLA_VERSION"] + def load_dropins() -> dict[str, t.Any]: return yaml.safe_load( - pathlib.Path("/opt/dropins.yml").read_text(encoding="utf-8") + pathlib.Path( + "/app/capella/versions", CAPELLA_VERSION, "dropins.yml" + ).read_text(encoding="utf-8") )["dropins"] @@ -44,7 +48,7 @@ def extract_repositories_and_install_ius(dropins: dict[str, t.Any]) -> None: def install_update_sites(repository: str, install_ui: list[str]) -> None: subprocess.run( [ - "/opt/capella/capella", + "/layers/capella/app/capella", "-consoleLog", "-application", "org.eclipse.equinox.p2.director", @@ -60,7 +64,8 @@ def install_update_sites(repository: str, install_ui: list[str]) -> None: def download_and_copy_dropin(download_url: str, file_name: str) -> None: urllib.request.urlretrieve( - download_url, pathlib.Path("/opt/capella/dropins") / file_name + download_url, + pathlib.Path("/layers/capella/app/dropins") / file_name, ) diff --git a/buildpacks/buildpacks/capella/autostart b/buildpacks/buildpacks/capella/autostart new file mode 100755 index 00000000..f29160cd --- /dev/null +++ b/buildpacks/buildpacks/capella/autostart @@ -0,0 +1,17 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +# Autostart script +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +if [ "$AUTOSTART_CAPELLA" = "1" ]; +then + if [ "$RESTART_CAPELLA" = "1" ]; + then + # Run capella in a loop: + ( while true; do /layers/capella/app/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log; sleep 1; done ) & + else + /layers/capella/app/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log & + fi +fi diff --git a/buildpacks/buildpacks/capella/bin/build b/buildpacks/buildpacks/capella/bin/build new file mode 100755 index 00000000..f689a00f --- /dev/null +++ b/buildpacks/buildpacks/capella/bin/build @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +CAPELLA_BUILD_TYPE=${CAPELLA_BUILD_TYPE:-offline} +BUILD_ARCHITECTURE=$(dpkg --print-architecture) +PATH_TO_CAPELLA_DIRECTORY=/app/capella/versions/$CAPELLA_VERSION/$BUILD_ARCHITECTURE + +BUILDPACK_CAPELLA_DIR=/app/buildpacks/buildpacks/capella + +if [ "$CAPELLA_BUILD_TYPE" = "online" ]; then + python3 ${BUILDPACK_CAPELLA_DIR}/download_archive.py $CAPELLA_VERSION; +else + cp /app/capella/versions/$CAPELLA_VERSION/$BUILD_ARCHITECTURE/capella.tar.gz /tmp/capella.tar.gz +fi + +mkdir /tmp/capella; +tar -xf /tmp/capella.tar.gz -C /tmp/capella; + +# Remove samples directory from archive and move to layer +mkdir ${CNB_LAYERS_DIR}/app +mv /tmp/capella/capella/* ${CNB_LAYERS_DIR}/app; + +# Some older archives don't have the correct execute permissions. Fix that. +chmod +x ${CNB_LAYERS_DIR}/app/capella +chmod -R +x ${CNB_LAYERS_DIR}/app/jre/bin +chmod -R +x ${CNB_LAYERS_DIR}/app/jre/lib + +# Create Eclipse settings directory +mkdir -p ${CNB_LAYERS_DIR}/app/configuration/.settings; + +# Do not show WORKSPACE_SELECTION_DIALOG +echo "SHOW_WORKSPACE_SELECTION_DIALOG=false" >> ${CNB_LAYERS_DIR}/app/configuration/.settings/org.eclipse.ui.ide.prefs; + +# Install patches +PATCH_DIR=/app/capella/versions/$CAPELLA_VERSION/patches /app/capella/patch.sh + +# Increase timeout and retries for better stability +echo '-Dorg.eclipse.equinox.p2.transport.ecf.retry=15' >> /layers/capella/app/capella.ini +echo '-Dorg.eclipse.ecf.provider.filetransfer.retrieve.readTimeout=10000' >> /layers/capella/app/capella.ini + +mkdir -p ${CNB_LAYERS_DIR}/meta +cp /app/capella/git_askpass.py ${CNB_LAYERS_DIR}/meta + +cp ${BUILDPACK_CAPELLA_DIR}/entrypoint.sh ${CNB_LAYERS_DIR}/meta +cp ${BUILDPACK_CAPELLA_DIR}/autostart ${CNB_LAYERS_DIR}/meta + +mkdir -p ${CNB_LAYERS_DIR}/meta/exec.d +cp ${BUILDPACK_CAPELLA_DIR}/exec.d/* ${CNB_LAYERS_DIR}/meta/exec.d +cp /app/eclipse/set_memory_flags.py ${CNB_LAYERS_DIR}/meta/exec.d +chmod +x ${CNB_LAYERS_DIR}/meta/exec.d/* + +# Set environment variables for build & run +mkdir -p ${CNB_LAYERS_DIR}/meta/env +echo -n "capella" > ${CNB_LAYERS_DIR}/meta/env/BASE_TYPE +echo -n "/layers/capella/app" > ${CNB_LAYERS_DIR}/meta/env/ECLIPSE_INSTALLATION_PATH +echo -n "/layers/capella/app/capella" > ${CNB_LAYERS_DIR}/meta/env/ECLIPSE_EXECUTABLE +echo -n "1" > ${CNB_LAYERS_DIR}/meta/env/AUTOSTART_CAPELLA +echo -n "1" > ${CNB_LAYERS_DIR}/meta/env/RESTART_CAPELLA + +cat > "${CNB_LAYERS_DIR}/meta.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/launch.toml" << EOL +[[processes]] +type = "capella" +command = ["${CNB_LAYERS_DIR}/meta/entrypoint.sh"] +default = true +EOL diff --git a/buildpacks/buildpacks/capella/bin/detect b/buildpacks/buildpacks/capella/bin/detect new file mode 100755 index 00000000..6a81c094 --- /dev/null +++ b/buildpacks/buildpacks/capella/bin/detect @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +CAPELLA_BUILD_TYPE=${CAPELLA_BUILD_TYPE:-offline} +BUILD_ARCHITECTURE=$(dpkg --print-architecture) +PATH_TO_CAPELLA_DIRECTORY=/app/capella/versions/$CAPELLA_VERSION/$BUILD_ARCHITECTURE + +if [[ -z $CAPELLA_VERSION ]]; then + echo "CAPELLA_VERSION is not set." + exit 100 +fi + +if [[ -z $BUILD_ARCHITECTURE ]]; then + echo "BUILD_ARCHITECTURE is not set." + exit 100 +fi + +if [ "$BUILD_ARCHITECTURE" != "amd64" ] && [ "$BUILD_ARCHITECTURE" != "arm64" ]; then + echo "BUILD_ARCHITECTURE must be one of 'amd64' or 'arm64'. Is '$BUILD_ARCHITECTURE'." + exit 100 +fi + +# Validate CAPELLA_BUILD_TYPE +if [ "$CAPELLA_BUILD_TYPE" != "online" ] && [ "$CAPELLA_BUILD_TYPE" != "offline" ]; then + echo "CAPELLA_BUILD_TYPE must be one of 'online' or 'offline'." + exit 100 +fi + +# Check if build mode is online, architecture is x86_64 (aarch64 not supported) +if [ "$BUILD_ARCHITECTURE" = "amd64" ] && [ "$CAPELLA_BUILD_TYPE" = "online" ]; then + echo "BUILD_ARCHITECTURE must be one of 'amd64' or 'arm64'." + exit 100 +fi + +# Check if build mode is online or capella.tar.gz exists +if [ "$CAPELLA_BUILD_TYPE" = "offline" ] && [[ ! -f $PATH_TO_CAPELLA_DIRECTORY/capella.tar.gz ]]; then + echo "Couldn't find capella/versions/$CAPELLA_VERSION/$BUILD_ARCHITECTURE/capella.tar.gz." + echo "Add the mentioned file or use '--env CAPELLA_BUILD_TYPE=online'." + exit 100 +fi + +# Check if CAPELLA_FORCE_REDOWNLOAD=1 is set + +# Validate the checksum of the capella.tar.gz diff --git a/buildpacks/buildpacks/capella/buildpack.toml b/buildpacks/buildpacks/capella/buildpack.toml new file mode 100644 index 00000000..2b3f7914 --- /dev/null +++ b/buildpacks/buildpacks/capella/buildpack.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "capella" +version = "0.0.0" + +[[targets]] +os = "linux" diff --git a/capella/download_archive.py b/buildpacks/buildpacks/capella/download_archive.py similarity index 96% rename from capella/download_archive.py rename to buildpacks/buildpacks/capella/download_archive.py index 94cbb1c1..2a4b3f95 100644 --- a/capella/download_archive.py +++ b/buildpacks/buildpacks/capella/download_archive.py @@ -57,6 +57,6 @@ def get_directory_structure(url: str) -> list[str]: download_response = requests.get(download_url) download_response.raise_for_status() - download_path = pathlib.Path("/opt/capella.tar.gz").write_bytes( + download_path = pathlib.Path("/tmp/capella.tar.gz").write_bytes( download_response.content ) diff --git a/remote/bg-saved.cfg b/buildpacks/buildpacks/capella/entrypoint.sh old mode 100644 new mode 100755 similarity index 65% rename from remote/bg-saved.cfg rename to buildpacks/buildpacks/capella/entrypoint.sh index 72f2f872..81631f19 --- a/remote/bg-saved.cfg +++ b/buildpacks/buildpacks/capella/entrypoint.sh @@ -1,7 +1,5 @@ +#!/bin/bash # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 -[xin_-1] -file=/tmp/wallpaper.png -mode=4 -bgcolor=#000000 +xvfb-run /layers/capella/app/capella "$@" diff --git a/capella/setup/provisioning.py b/buildpacks/buildpacks/capella/exec.d/provisioning.py old mode 100644 new mode 100755 similarity index 98% rename from capella/setup/provisioning.py rename to buildpacks/buildpacks/capella/exec.d/provisioning.py index 7d2c12e5..28f498f2 --- a/capella/setup/provisioning.py +++ b/buildpacks/buildpacks/capella/exec.d/provisioning.py @@ -1,3 +1,5 @@ +#!/layers/venv/venv/bin/python + # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 """Prepare models in the $MODEL_INBOX_DIRECTORIES for Capella.""" @@ -165,9 +167,7 @@ def provide_project_dirs_to_capella_plugin( projects: list[_ProjectDict], ) -> None: locations = ":".join([str(project["location"]) for project in projects]) - pathlib.Path("/etc/environment").write_text( - f"export MODEL_INBOX_DIRECTORIES={locations}\n", encoding="utf-8" - ) + os.write(3, f'MODEL_INBOX_DIRECTORIES="{locations}"\n'.encode()) log.info( "Set environment variable MODEL_INBOX_DIRECTORIES to '%s'", locations ) diff --git a/buildpacks/buildpacks/capella/exec.d/set_autostart.sh b/buildpacks/buildpacks/capella/exec.d/set_autostart.sh new file mode 100755 index 00000000..e31cb71a --- /dev/null +++ b/buildpacks/buildpacks/capella/exec.d/set_autostart.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p /home/techuser/.config/openbox +ln -s /layers/capella/meta/autostart /home/techuser/.config/openbox/autostart diff --git a/capella/setup/setup_workspace.py b/buildpacks/buildpacks/capella/exec.d/setup_workspace.py old mode 100644 new mode 100755 similarity index 99% rename from capella/setup/setup_workspace.py rename to buildpacks/buildpacks/capella/exec.d/setup_workspace.py index eac967b6..eafbf4a6 --- a/capella/setup/setup_workspace.py +++ b/buildpacks/buildpacks/capella/exec.d/setup_workspace.py @@ -1,3 +1,5 @@ +#!/usr/bin/python + # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/buildpacks/buildpacks/supervisord/bin/build b/buildpacks/buildpacks/supervisord/bin/build new file mode 100755 index 00000000..5095abc8 --- /dev/null +++ b/buildpacks/buildpacks/supervisord/bin/build @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p ${CNB_LAYERS_DIR}/app + +python3 -m venv ${CNB_LAYERS_DIR}/app/venv +${CNB_LAYERS_DIR}/app/venv/bin/pip install --no-cache-dir supervisor + +cat > "${CNB_LAYERS_DIR}/app/supervisord.conf" << EOL +[supervisord] +nodaemon=true +childlogdir=/layers/supervisord +EOL + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/launch.toml" << EOL +[[processes]] +type = "supervisord" +command = ["${CNB_LAYERS_DIR}/app/venv/bin/supervisord", "-c", "${CNB_LAYERS_DIR}/app/supervisord.conf"] +default = true +EOL diff --git a/buildpacks/buildpacks/supervisord/bin/detect b/buildpacks/buildpacks/supervisord/bin/detect new file mode 100755 index 00000000..2491109f --- /dev/null +++ b/buildpacks/buildpacks/supervisord/bin/detect @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail diff --git a/buildpacks/buildpacks/supervisord/buildpack.toml b/buildpacks/buildpacks/supervisord/buildpack.toml new file mode 100644 index 00000000..bfb20f68 --- /dev/null +++ b/buildpacks/buildpacks/supervisord/buildpack.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "supervisord" +version = "0.0.0" +description = "Install the supervisord process manager." + +[[targets]] +os = "linux" diff --git a/buildpacks/buildpacks/t4c/bin/build b/buildpacks/buildpacks/t4c/bin/build new file mode 100755 index 00000000..cc858251 --- /dev/null +++ b/buildpacks/buildpacks/t4c/bin/build @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p ${CNB_LAYERS_DIR}/meta +cp /app/buildpacks/buildpacks/t4c/entrypoint.sh ${CNB_LAYERS_DIR}/meta + +mkdir -p ${CNB_LAYERS_DIR}/meta/exec.d +cp /app/buildpacks/buildpacks/t4c/exec.d/* ${CNB_LAYERS_DIR}/meta/exec.d +chmod +x ${CNB_LAYERS_DIR}/meta/exec.d/* + +# Compute a diff (all files created when installing the T4C extension) +capella_layer=/layers/capella/app +capella_tmp_layer=$(mktemp -d) +t4c_diff_layer=${CNB_LAYERS_DIR}/t4c + +# Snapshot before installation of T4C extension +rsync -a ${capella_layer}/ ${capella_tmp_layer}/ + +# Install T4C client +T4C_ZIP=$(find /app/t4c/updateSite/$CAPELLA_VERSION -type f -iname "com.thalesgroup.mde.melody.team.license.update-*.zip" | head -n 1); \ +${capella_tmp_layer}/capella \ + -clean \ + -data $(mktemp -d) \ + -consoleLog \ + -application org.eclipse.equinox.p2.director \ + -noSplash \ + -repository jar:file://${T4C_ZIP}!/ \ + -installIU com.thalesgroup.mde.melody.collab.feature.feature.group,com.thalesgroup.mde.melody.collab.maintenance.feature.feature.group,com.thalesgroup.mde.melody.collab.licbranding.feature.feature.group + +# Install patches +PATCH_DIR=/app/t4c/updateSite/$CAPELLA_VERSION /app/capella/patch.sh + +# Compute diff +rsync -a --compare-dest=${capella_layer}/ ${capella_tmp_layer}/ ${t4c_diff_layer}/ + +# Install T4C CLI +mkdir -p ${CNB_LAYERS_DIR}/cli +python3 -m venv ${CNB_LAYERS_DIR}/cli + +cat > "${CNB_LAYERS_DIR}/t4c.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/meta.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/cli.toml" << EOL +[types] +launch = true +EOL + +cat > "${CNB_LAYERS_DIR}/launch.toml" << EOL +[[processes]] +type = "t4c" +command = ["${CNB_LAYERS_DIR}/meta/entrypoint.sh"] +default = true +EOL diff --git a/buildpacks/buildpacks/t4c/bin/detect b/buildpacks/buildpacks/t4c/bin/detect new file mode 100755 index 00000000..cc20abcb --- /dev/null +++ b/buildpacks/buildpacks/t4c/bin/detect @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +if [[ "$INSTALL_T4C_CLIENT" != "1" ]]; then + echo "Skipping installation of T4C client, \$INSTALL_T4C_CLIENT is not set to 1." + exit 100 +fi + +if find /app/t4c/updateSite/$CAPELLA_VERSION -type f -iname "com.thalesgroup.mde.melody.team.license.update-*.zip" | head -n 1; then + echo "Couldn't find com.thalesgroup.mde.melody.team.license.update-*.zip in /app/t4c/updateSite/$CAPELLA_VERSION." + exit 1 +fi diff --git a/buildpacks/buildpacks/t4c/buildpack.toml b/buildpacks/buildpacks/t4c/buildpack.toml new file mode 100644 index 00000000..d8b93288 --- /dev/null +++ b/buildpacks/buildpacks/t4c/buildpack.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "t4c" +version = "0.0.0" + +[[targets]] +os = "linux" diff --git a/t4c/docker_entrypoint.sh b/buildpacks/buildpacks/t4c/entrypoint.sh similarity index 69% rename from t4c/docker_entrypoint.sh rename to buildpacks/buildpacks/t4c/entrypoint.sh index e2da1d68..862f8491 100755 --- a/t4c/docker_entrypoint.sh +++ b/buildpacks/buildpacks/t4c/entrypoint.sh @@ -7,15 +7,15 @@ set -euo pipefail case ${1:-startup} in backup) - xvfb-run /opt/t4c_cli/.venv/bin/backup + xvfb-run /layers/t4c/cli/bin/backup ;; export) - xvfb-run /opt/t4c_cli/.venv/bin/exporter + xvfb-run /layers/t4c/cli/bin/exporter ;; startup) - /startup.sh + /layers/capella/meta/entrypoint.sh ;; *) diff --git a/buildpacks/buildpacks/t4c/exec.d/inject_in_capella_ini.sh b/buildpacks/buildpacks/t4c/exec.d/inject_in_capella_ini.sh new file mode 100755 index 00000000..59e884a4 --- /dev/null +++ b/buildpacks/buildpacks/t4c/exec.d/inject_in_capella_ini.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +cat >> /layers/capella/app/capella.ini <> "/layers/supervisord/app/supervisord.conf" << EOL +[program:idletime] +command=/layers/xidletime/app/venv/bin/python /layers/xidletime/app/metrics.py +user=techuser +autorestart=true +environment=DISPLAY=":10" +EOL + +mkdir -p ${CNB_LAYERS_DIR}/app + +python3 -m venv ${CNB_LAYERS_DIR}/app/venv +${CNB_LAYERS_DIR}/app/venv/bin/pip install --no-cache-dir prometheus-client + +cp /app/buildpacks/buildpacks/xidletime/metrics.py ${CNB_LAYERS_DIR}/app/metrics.py + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL diff --git a/buildpacks/buildpacks/xidletime/bin/detect b/buildpacks/buildpacks/xidletime/bin/detect new file mode 100755 index 00000000..2491109f --- /dev/null +++ b/buildpacks/buildpacks/xidletime/bin/detect @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail diff --git a/buildpacks/buildpacks/xidletime/buildpack.toml b/buildpacks/buildpacks/xidletime/buildpack.toml new file mode 100644 index 00000000..2887806e --- /dev/null +++ b/buildpacks/buildpacks/xidletime/buildpack.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "xidletime" +version = "0.0.0" +description = "Measure the idle time of the XServer and expose it via a metrics endpoint." + +[[targets]] +os = "linux" diff --git a/remote/metrics.py b/buildpacks/buildpacks/xidletime/metrics.py similarity index 100% rename from remote/metrics.py rename to buildpacks/buildpacks/xidletime/metrics.py diff --git a/buildpacks/buildpacks/xpra/bin/build b/buildpacks/buildpacks/xpra/bin/build new file mode 100755 index 00000000..235edf87 --- /dev/null +++ b/buildpacks/buildpacks/xpra/bin/build @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p ${CNB_LAYERS_DIR}/app/exec.d +cp /app/buildpacks/buildpacks/xpra/exec.d/* ${CNB_LAYERS_DIR}/app/exec.d +chmod +x ${CNB_LAYERS_DIR}/app/exec.d/* + +cp /app/buildpacks/buildpacks/xpra/nginx.conf ${CNB_LAYERS_DIR}/app/nginx.conf +mkdir -p ${CNB_LAYERS_DIR}/app/html +cp /app/buildpacks/buildpacks/xpra/error.html ${CNB_LAYERS_DIR}/app/html/error.html + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL diff --git a/buildpacks/buildpacks/xpra/bin/detect b/buildpacks/buildpacks/xpra/bin/detect new file mode 100755 index 00000000..2491109f --- /dev/null +++ b/buildpacks/buildpacks/xpra/bin/detect @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail diff --git a/buildpacks/buildpacks/xpra/buildpack.toml b/buildpacks/buildpacks/xpra/buildpack.toml new file mode 100644 index 00000000..a556a74e --- /dev/null +++ b/buildpacks/buildpacks/xpra/buildpack.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "xpra" +version = "0.0.0" +description = "Install xpra-html5 to run X11 applications in a browser." + +[[targets]] +os = "linux" diff --git a/remote/error.html b/buildpacks/buildpacks/xpra/error.html similarity index 100% rename from remote/error.html rename to buildpacks/buildpacks/xpra/error.html diff --git a/remote/supervisord.xpra.conf b/buildpacks/buildpacks/xpra/exec.d/add_supervisord_config.sh old mode 100644 new mode 100755 similarity index 72% rename from remote/supervisord.xpra.conf rename to buildpacks/buildpacks/xpra/exec.d/add_supervisord_config.sh index e43d4a4a..54887469 --- a/remote/supervisord.xpra.conf +++ b/buildpacks/buildpacks/xpra/exec.d/add_supervisord_config.sh @@ -1,6 +1,10 @@ +#!/bin/bash + # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 +if [ "${CONNECTION_METHOD:-xrdp}" == "xpra" ]; then + cat >> "/layers/supervisord/app/supervisord.conf" << EOL [program:xpra] command=xpra start :10 --start=/home/techuser/.config/openbox/autostart --start-env=GTK_IM_MODULE=ibus --attach=yes --daemon=no --bind-tcp=0.0.0.0:10001 --min-quality=70 user=techuser @@ -8,6 +12,8 @@ autorestart=true environment=DISPLAY=":10",XPRA_DEFAULT_CONTENT_TYPE="text",XPRA_DEFAULT_VFB_RESOLUTION="1920x1080" [program:nginx] -command=nginx +command=nginx -c /layers/xpra/app/nginx.conf user=techuser autorestart=true +EOL +fi diff --git a/buildpacks/buildpacks/xpra/exec.d/replace_nginx_vars.sh b/buildpacks/buildpacks/xpra/exec.d/replace_nginx_vars.sh new file mode 100755 index 00000000..4daed0ce --- /dev/null +++ b/buildpacks/buildpacks/xpra/exec.d/replace_nginx_vars.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +# Replace Variables in the nginx.conf +sed -i "s|__XPRA_SUBPATH__|${XPRA_SUBPATH:-/}|g" /layers/xpra/app/nginx.conf +sed -i "s|__XPRA_CSP_ORIGIN_HOST__|${XPRA_CSP_ORIGIN_HOST:-}|g" /layers/xpra/app/nginx.conf diff --git a/remote/nginx.conf b/buildpacks/buildpacks/xpra/nginx.conf similarity index 96% rename from remote/nginx.conf rename to buildpacks/buildpacks/xpra/nginx.conf index 371ae1cc..1116f14a 100644 --- a/remote/nginx.conf +++ b/buildpacks/buildpacks/xpra/nginx.conf @@ -16,7 +16,7 @@ http { listen 10000; server_name _; - root /usr/share/nginx/html; + root /layers/xpra/app/html; error_page 502 /error.html; error_page 404 /error.html; diff --git a/buildpacks/buildpacks/xrdp/bin/build b/buildpacks/buildpacks/xrdp/bin/build new file mode 100755 index 00000000..70203bfd --- /dev/null +++ b/buildpacks/buildpacks/xrdp/bin/build @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +mkdir -p ${CNB_LAYERS_DIR}/app/exec.d +cp /app/buildpacks/buildpacks/xrdp/exec.d/* ${CNB_LAYERS_DIR}/app/exec.d +chmod +x ${CNB_LAYERS_DIR}/app/exec.d/* + +mkdir -p ${CNB_LAYERS_DIR}/app +cp /app/buildpacks/buildpacks/xrdp/rc.xml ${CNB_LAYERS_DIR}/app/rc.xml +cp /app/buildpacks/buildpacks/xrdp/menu.xml ${CNB_LAYERS_DIR}/app/menu.xml + +cat > "${CNB_LAYERS_DIR}/app.toml" << EOL +[types] +launch = true +EOL diff --git a/buildpacks/buildpacks/xrdp/bin/detect b/buildpacks/buildpacks/xrdp/bin/detect new file mode 100755 index 00000000..2491109f --- /dev/null +++ b/buildpacks/buildpacks/xrdp/bin/detect @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail diff --git a/buildpacks/buildpacks/xrdp/buildpack.toml b/buildpacks/buildpacks/xrdp/buildpack.toml new file mode 100644 index 00000000..fb866c8d --- /dev/null +++ b/buildpacks/buildpacks/xrdp/buildpack.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[buildpack] +id = "xrdp" +version = "0.0.0" +description = "Install xrdp to expose X11 applications via RDP." + +[[targets]] +os = "linux" diff --git a/remote/supervisord.xrdp.conf b/buildpacks/buildpacks/xrdp/exec.d/add_supervisord_config.sh old mode 100644 new mode 100755 similarity index 66% rename from remote/supervisord.xrdp.conf rename to buildpacks/buildpacks/xrdp/exec.d/add_supervisord_config.sh index 384c7d9c..64cf35e2 --- a/remote/supervisord.xrdp.conf +++ b/buildpacks/buildpacks/xrdp/exec.d/add_supervisord_config.sh @@ -1,13 +1,20 @@ +#!/bin/bash + # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 +if [ "${CONNECTION_METHOD:-xrdp}" == "xrdp" ]; then + cat >> "/layers/supervisord/app/supervisord.conf" << EOL [program:xrdp] command=/usr/sbin/xrdp --nodaemon user=techuser autorestart=true [program:xrdp-sesman] +directory=/home/techuser command=/usr/sbin/xrdp-sesman --nodaemon user=techuser autorestart=true environment=DISPLAY=":10" +EOL +fi diff --git a/buildpacks/buildpacks/xrdp/exec.d/copy_openbox_config.sh b/buildpacks/buildpacks/xrdp/exec.d/copy_openbox_config.sh new file mode 100755 index 00000000..81c36eb2 --- /dev/null +++ b/buildpacks/buildpacks/xrdp/exec.d/copy_openbox_config.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +cp /layers/xrdp/app/rc.xml /etc/xdg/openbox/rc.xml +cp /layers/xrdp/app/menu.xml /etc/xdg/openbox/menu.xml diff --git a/buildpacks/buildpacks/xrdp/exec.d/set_password.sh b/buildpacks/buildpacks/xrdp/exec.d/set_password.sh new file mode 100755 index 00000000..83a7f129 --- /dev/null +++ b/buildpacks/buildpacks/xrdp/exec.d/set_password.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +if [ "${CONNECTION_METHOD:-xrdp}" == "xrdp" ]; +then + if [ "$(whoami)" == "root" ] || [ "$(whoami)" == "techuser" ]; + then + salt=$(openssl rand -base64 16) + password_hash=$(openssl passwd -6 -salt ${salt} "${RMT_PASSWORD:?}") + line=$(grep techuser /etc/shadow); + echo ${line%%:*}:${password_hash}:${line#*:*:} > /etc/shadow; + else + echo "Only techuser and root are supported as users."; + exit 1; + fi +fi diff --git a/remote/menu.xml b/buildpacks/buildpacks/xrdp/menu.xml similarity index 100% rename from remote/menu.xml rename to buildpacks/buildpacks/xrdp/menu.xml diff --git a/remote/rc.xml b/buildpacks/buildpacks/xrdp/rc.xml similarity index 100% rename from remote/rc.xml rename to buildpacks/buildpacks/xrdp/rc.xml diff --git a/buildpacks/extensions/capella-deps/bin/detect b/buildpacks/extensions/capella-deps/bin/detect new file mode 100755 index 00000000..2491109f --- /dev/null +++ b/buildpacks/extensions/capella-deps/bin/detect @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail diff --git a/buildpacks/extensions/capella-deps/bin/generate b/buildpacks/extensions/capella-deps/bin/generate new file mode 100755 index 00000000..6a2caf30 --- /dev/null +++ b/buildpacks/extensions/capella-deps/bin/generate @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +cat >> "${CNB_OUTPUT_DIR}/run.Dockerfile" <> "${CNB_OUTPUT_DIR}/run.Dockerfile" +fi + + +cat >> "${CNB_OUTPUT_DIR}/run.Dockerfile" <> "${CNB_OUTPUT_DIR}/run.Dockerfile" <> "${CNB_OUTPUT_DIR}/extend-config.toml" <> "${CNB_OUTPUT_DIR}/run.Dockerfile" <> "${CNB_OUTPUT_DIR}/run.Dockerfile" < /etc/X11/Xwrapper.config && \ + chmod 666 /etc/shadow && \ + mkdir -p /run/xrdp/sockdir && \ + chown -R techuser /etc/xrdp /run/xrdp /var/log/xrdp* /etc/xdg/openbox + +ARG user_id +USER \${user_id} +EOL diff --git a/buildpacks/extensions/xrdp-deps/extension.toml b/buildpacks/extensions/xrdp-deps/extension.toml new file mode 100644 index 00000000..250662e2 --- /dev/null +++ b/buildpacks/extensions/xrdp-deps/extension.toml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +api = "0.11" + +[extension] +id = "xrdp-deps" +name = "Xrdp system dependencies" +version = "0.0.0" diff --git a/capella/.dockerignore.template b/capella/.dockerignore.template deleted file mode 100644 index 851fa735..00000000 --- a/capella/.dockerignore.template +++ /dev/null @@ -1,7 +0,0 @@ -# Please only edit the .dockerignore.template file! -# The .dockerignore file is gitignored and updated by the Makefile -# Environment variables are replaced before build with `envsubst` - -.dockerignore.template -versions/* -!versions/$CAPELLA_VERSION diff --git a/capella/.dockerignore.template.license b/capella/.dockerignore.template.license deleted file mode 100644 index 7ea22469..00000000 --- a/capella/.dockerignore.template.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -SPDX-License-Identifier: Apache-2.0 diff --git a/capella/Dockerfile b/capella/Dockerfile deleted file mode 100644 index b3945406..00000000 --- a/capella/Dockerfile +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -ARG BASE_IMAGE=base -ARG BUILD_TYPE=online -ARG INSTALL_OLD_GTK_VERSION=true -FROM ${BASE_IMAGE} as base - -ENV DEBIAN_FRONTEND=noninteractive - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -ENV SHELL=/bin/bash - -FROM base as old_gtk_true - -ONBUILD USER root - -# Install WebKit with GTK -ONBUILD COPY libs /tmp/libs -ONBUILD ARG INJECT_PACKAGES=false -# hadolint ignore=SC2046 -ONBUILD RUN if [ "$INJECT_PACKAGES" = "true" ]; then \ - apt-get update && \ - # Inject old packages manually - apt-get install -y $(find /tmp/libs -iname "*.deb"); \ - rm -rf /var/lib/apt/lists/*; \ - rm -r /tmp/libs; \ - else \ - # Download old packages from the Ubuntu focal registry - ## Add source - echo "deb http://de.archive.ubuntu.com/ubuntu/ focal main" >> /etc/apt/sources.list.d/focal.list; \ - ## Import the required keys - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32; \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 871920D1991BC93C; \ - apt-get update; \ - ## Install the packages - apt-get install -y \ - libjavascriptcoregtk-4.0-18=2.28.1-1 \ - libwebkit2gtk-4.0-37=2.28.1-1; \ - rm /etc/apt/sources.list.d/focal.list; \ - fi; \ - rm -rf /var/lib/apt/lists/*; - -FROM base as old_gtk_false - -ONBUILD USER root -ONBUILD COPY libs /tmp/libs -ONBUILD RUN apt-get update && \ - apt-get install -y \ - libjavascriptcoregtk-4.0-18 \ - libwebkit2gtk-4.0-37 && \ - rm -rf /var/lib/apt/lists/*; - -FROM old_gtk_${INSTALL_OLD_GTK_VERSION} as base_new - -WORKDIR /opt/ - -FROM base_new as build_online -# Download a Capella executable archive - -# https://github.com/moby/moby/issues/26533#issuecomment-246966836 -ONBUILD ARG CAPELLA_VERSION="5.2.0" -ONBUILD ARG BUILD_ARCHITECTURE="amd64" -ONBUILD COPY ./download_archive.py /opt/.download_archive.py -ONBUILD RUN pip install --no-cache-dir requests && \ - python .download_archive.py ${CAPELLA_VERSION}; - -FROM base_new as build_offline - -# ONBUILD is required here -# https://github.com/moby/moby/issues/26533#issuecomment-246966836 -ONBUILD ARG CAPELLA_VERSION="5.2.0" -ONBUILD ARG BUILD_ARCHITECTURE="amd" -ONBUILD COPY ./versions/${CAPELLA_VERSION}/${BUILD_ARCHITECTURE}/capella.* /opt/ - - -FROM build_${BUILD_TYPE} - -ARG CAPELLA_VERSION - -RUN apt-get update && \ - apt-get install -y \ - libxtst6 \ - xdg-utils \ - xvfb \ - xauth \ - dbus-x11 && \ - rm -rf /var/lib/apt/lists/*; - -RUN if [ -s capella.zip ]; then \ - # Install Capella using zip - unzip capella.zip -d . && \ - rm capella.zip; \ - # Install Capella using tar.gz - elif [ -s capella.tar.gz ]; then \ - tar -xf capella.tar.gz; \ - else \ - echo "capella.zip and capella.tar.gz are empty! Please add you custom Capella zip or tar.gz" && \ - exit 1; \ - fi && \ - rm -rf samples capella.zip capella.tar.gz - -# Set Permissions -RUN chmod +x capella/capella && \ - chmod -R +x capella/jre/bin && \ - chmod -R +x capella/jre/lib && \ - chown -R techuser /opt/capella - -COPY patch.sh /opt/patch.sh -RUN chmod +x /opt/patch.sh - -# Install Dropins for Capella -USER techuser - -COPY ./versions/${CAPELLA_VERSION}/dropins /opt/capella/dropins -ARG CAPELLA_DROPINS="" -COPY install_dropins.py /opt/install_dropins.py -COPY ./versions/${CAPELLA_VERSION}/dropins.yml /opt/dropins.yml - -ARG MEMORY_LIMIT=5500m - -RUN echo '-Dorg.eclipse.equinox.p2.transport.ecf.retry=15' >> /opt/capella/capella.ini && \ - echo '-Dorg.eclipse.ecf.provider.filetransfer.retrieve.readTimeout=10000' >> /opt/capella/capella.ini && \ - sed -i "s/-Xmx[^ ]*/-Xmx$MEMORY_LIMIT/g" /opt/capella/capella.ini -RUN python install_dropins.py - -COPY ./versions/${CAPELLA_VERSION}/patches /opt/patches -RUN PATCH_DIR=/opt/patches /opt/patch.sh - -USER root - -# Eclipse settings -RUN mkdir -p /opt/capella/configuration/.settings; \ - ## Do not show WORKSPACE_SELECTION_DIALOG - echo "SHOW_WORKSPACE_SELECTION_DIALOG=false" >> /opt/capella/configuration/.settings/org.eclipse.ui.ide.prefs; \ - # Setup workspace - mkdir -p /workspace; \ - # Set workspace permissions - chown -R techuser /workspace && \ - chmod -R 777 /opt/capella/configuration && \ - chmod -R 777 /opt/capella/p2/org.eclipse.equinox.p2.engine/profileRegistry && \ - chown techuser /opt /opt/capella/capella.ini - -RUN echo '-Dosgi.configuration.area=file:/opt/capella/configuration' >> /opt/capella/capella.ini -COPY setup/* /opt/setup/ - -ENV AUTOSTART_CAPELLA=1 -ENV RESTART_CAPELLA=1 -COPY ./autostart /home/techuser/.config/openbox/autostart - -ENV ECLIPSE_INSTALLATION_PATH=/opt/capella -ENV ECLIPSE_EXECUTABLE=/opt/capella/capella - -# Set memory options for the JVM (used by set_memory_flags.py) -ARG MEMORY_MAX=90% -ENV MEMORY_MAX=${MEMORY_MAX} -ARG MEMORY_MIN=70% -ENV MEMORY_MIN=${MEMORY_MIN} - -COPY ./autostart /home/techuser/.config/openbox/autostart - -COPY git_askpass.py /etc/git_askpass.py -RUN chmod 555 /etc/git_askpass.py - -ARG BUILD_ARCHITECTURE=amd64 -ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini-$BUILD_ARCHITECTURE /tini -RUN chmod +x /tini - -COPY startup.sh /startup.sh -ENTRYPOINT [ "/tini", "--", "/startup.sh" ] - -ENV BASE_TYPE=capella - -USER techuser diff --git a/capella/autostart b/capella/autostart deleted file mode 100755 index fc41979e..00000000 --- a/capella/autostart +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -# Autostart script -export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -# Load environment variables from /etc/environment -source /etc/environment - -nitrogen --restore & - -if [ "$AUTOSTART_CAPELLA" = "1" ]; -then - if [ "$RESTART_CAPELLA" = "1" ]; - then - # Run capella in a loop: - ( while true; do /opt/capella/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log; sleep 1; done ) & - else - /opt/capella/capella -data ${WORKSPACE_DIR:-/workspace} > /var/log/capella.stdout.log 2> /var/log/capella.stderr.log & - fi -fi diff --git a/capella/libs/.gitkeep b/capella/libs/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/capella/patch.sh b/capella/patch.sh index 1c4b94fe..9186475d 100755 --- a/capella/patch.sh +++ b/capella/patch.sh @@ -2,13 +2,15 @@ # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 -if [[ -n "${PATCH_DIR:?}/patch_info.csv" ]]; +if [[ -f "${PATCH_DIR:?}/patch_info.csv" ]]; then while IFS="," read -r patch_zip install_iu tag do INSTALL_IU_JOIN=$(echo $install_iu | sed "s/ /,/g"); - /opt/capella/capella \ + /layers/capella/app/capella \ -consoleLog \ + -clean \ + -data $(mktemp -d) \ -application org.eclipse.equinox.p2.director \ -profile DefaultProfile \ -tag $tag \ diff --git a/capella/setup/README.md b/capella/setup/README.md deleted file mode 100644 index 1d7f876b..00000000 --- a/capella/setup/README.md +++ /dev/null @@ -1,9 +0,0 @@ - - -# Directory for startup scripts - -This directory contains scripts that are executed during container startup. As -of now, we support scripts with file-ending `.sh` and `.py`. diff --git a/capella/setup/replace_env_variables.sh b/capella/setup/replace_env_variables.sh deleted file mode 100755 index 21d456f6..00000000 --- a/capella/setup/replace_env_variables.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -# Replace environment variables in capella.ini, e.g. licences -envsubst < /opt/capella/capella.ini > /tmp/capella.ini && mv /tmp/capella.ini /opt/capella/capella.ini; diff --git a/capella/startup.sh b/capella/startup.sh deleted file mode 100755 index 93c0626d..00000000 --- a/capella/startup.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -export DISPLAY=":99" -python3 /opt/setup/set_memory_flags.py -(Xvfb ${DISPLAY} -screen 0 1920x1080x8 -nolisten tcp 0>/dev/null 2>&1 &) -/opt/capella/capella "$@" diff --git a/ci-templates/gitlab/diagram-cache.yml b/ci-templates/gitlab/diagram-cache.yml index f90421b1..f277403c 100644 --- a/ci-templates/gitlab/diagram-cache.yml +++ b/ci-templates/gitlab/diagram-cache.yml @@ -7,15 +7,13 @@ variables: update_capella_diagram_cache: image: name: $DOCKER_REGISTRY/capella/base:${CAPELLA_DOCKER_IMAGES_TAG} - entrypoint: [""] + entrypoint: [''] script: - - pip install "capellambse[cli]@git+https://github.com/DSD-DBS/py-capellambse.git@${CAPELLAMBSE_REVISION}" + - pip install + "capellambse[cli]@git+https://github.com/DSD-DBS/py-capellambse.git@${CAPELLAMBSE_REVISION}" - mkdir diagram_cache - - xvfb-run python -m capellambse.diagram_cache - --model "${ENTRYPOINT}" - --output ./diagram_cache - --index - --exe "/opt/capella/capella" + - xvfb-run python -m capellambse.diagram_cache --model "${ENTRYPOINT}" + --output ./diagram_cache --index --exe "/layers/capella/app/capella" - | if [[ "${PUSH_DIAGRAM_CACHE}" == "false" ]]; then exit 0 @@ -27,11 +25,11 @@ update_capella_diagram_cache: git push -o ci.skip --set-upstream origin ${DERIVED_BRANCH_NAME} --force artifacts: paths: - - "diagram_cache" + - 'diagram_cache' variables: # Virtual display used to run Capella in the background. # Do not modify the value! - DISPLAY: ":99" + DISPLAY: ':99' # Docker tag, which is used for the capella/base image. # Defaults to ${CAPELLA_VERSION}-${CAPELLA_DOCKER_IMAGES_REVISION} if not defined @@ -39,15 +37,15 @@ update_capella_diagram_cache: # Whether to push the diagram cache changes or not # Defaults to false if not defined - PUSH_DIAGRAM_CACHE: "false" + PUSH_DIAGRAM_CACHE: 'false' # The commit message used for updating the diagram cache # Defaults to "chore: Update diagram cache" if not defined - COMMIT_MSG: "chore: Update diagram cache" + COMMIT_MSG: 'chore: Update diagram cache' # The revision of the py-capellambse package to install # Defaults to "master" if not defined - CAPELLAMBSE_REVISION: "master" + CAPELLAMBSE_REVISION: 'master' # Relative entrypoint to .aird file inside repository (starting from the root of the repository). # Example: test/test.aird diff --git a/ci-templates/gitlab/model-validation.yml b/ci-templates/gitlab/model-validation.yml index ff728e48..8b715453 100644 --- a/ci-templates/gitlab/model-validation.yml +++ b/ci-templates/gitlab/model-validation.yml @@ -4,34 +4,36 @@ model-validation: image: name: $DOCKER_REGISTRY/capella/base:${CAPELLA_DOCKER_IMAGES_TAG} - entrypoint: [""] + entrypoint: [''] script: # As our containers run with techuser (non-root) git commands fail with dubious ownership, see for the solution. - git config --global --add safe.directory ${CI_PROJECT_DIR} - git fetch - - if [ ! -z $CI_COMMIT_BRANCH ]; then git reset --hard origin/$CI_COMMIT_BRANCH; fi + - if [ ! -z $CI_COMMIT_BRANCH ]; then git reset --hard + origin/$CI_COMMIT_BRANCH; fi - BUILD_DIR=$(pwd) - - if [ ! -f ".project" ]; then echo "The model validator requires a '.project' file!" && exit 1; fi - - 'if [[ "$(dirname "${ENTRYPOINT}")" != "." ]]; then echo ''The model validator requires a "*.aird" in the root level of the repository!'' && exit 1; fi' + - if [ ! -f ".project" ]; then echo "The model validator requires a + '.project' file!" && exit 1; fi + - 'if [[ "$(dirname "${ENTRYPOINT}")" != "." ]]; then echo ''The model + validator requires a "*.aird" in the root level of the repository!'' && + exit 1; fi' - mkdir /tmp/project - cp -r . /tmp/project - cd /tmp - > - xvfb-run /opt/capella/capella - -nosplash - -application org.polarsys.capella.core.commandline.core - -appid org.polarsys.capella.core.validation.commandline - -data "$(pwd)" - -input "project/$ENTRYPOINT" - -outputfolder "project/validation" - - cp "/tmp/project/validation/project/$ENTRYPOINT/validation-results.html" $BUILD_DIR + xvfb-run /layers/capella/app/capella -nosplash -application + org.polarsys.capella.core.commandline.core -appid + org.polarsys.capella.core.validation.commandline -data "$(pwd)" -input + "project/$ENTRYPOINT" -outputfolder "project/validation" + - cp "/tmp/project/validation/project/$ENTRYPOINT/validation-results.html" + $BUILD_DIR artifacts: paths: - - "validation-results.html" + - 'validation-results.html' variables: # Virtual display used to run Capella in the background. # Do not modify the value! - DISPLAY: ":99" + DISPLAY: ':99' # Docker tag, which is used for the capella/base image. # Defaults to ${CAPELLA_VERSION}-${CAPELLA_DOCKER_IMAGES_REVISION} if not defined diff --git a/docs/docs/design-decisions/buildpacks.md b/docs/docs/design-decisions/buildpacks.md new file mode 100644 index 00000000..404522e4 --- /dev/null +++ b/docs/docs/design-decisions/buildpacks.md @@ -0,0 +1,61 @@ + + +# Build Images with Cloud Native Buildpacks + +Starting with version 3.0.0, we use Cloud Native Buildpacks for the Capella +based images. + +## Why Cloud Native Buildpacks? + +### Disadvantages of Dockerfiles + +Before v3.0.0, we have used Dockerfiles to build the images. To keep the system +dependencies up to date, we have regularly rebuilt the images without build +cache. + +In combination with bad use of multi-stage builds, this has led to enormous +image sizes and long build times. Image layers were not cached at all. Our +internal Docker registry has grown to several terabytes in size. + +In addition, pulling the images from the registry was slow and after updates, +all layers had to be pulled again. + +### Advantages of Buildpacks for our Images + +- Exchange individual layers. Some examples where this is useful: + - Update the dependencies without update of the Capella layer + - Update TeamForCapella archive without update of the Capella layer + - Update dropins independently +- Reduction of the image sizes = faster image pull times = faster session + startup in the Collaboration Manager. Examples: + - `capella/base:6.0.0`: 3.5GB -> 1.5GB + - `t4c/client/base:6.0.0`: 4.5GB -> 1.6GB +- Reuse layers of last builds, massively reducing storage space and traffic in + registries Less layers, the new layers are clearly mentioned and separated in + different scopes (capella layer, capella-dropins layer, t4c-client layer, + ...) +- Full reproducible builds (same input = same digest of output images) +- No extensive build cache which sums up in local development (instead, it uses + layers of the last image as cache) +- It makes the images modular. Users can decide which modules they want to have + in their images (e.g. add xpra or xrdp as add-on to get remote support) +- Reduce number of layers in the images. + +### Two layers: Meta and App + +To have consistent naming, we've decided that we want to allow each buildpack +to define two layers: + +- `meta` layer: contains `exec.d` scripts, entrypoint scripts, and other + customizations which change frequently. +- `app` layer: contains the application (e. g. Capella, Eclipse, ...) and other + files which change less frequently. As the app layer is larger in size, we + want to cache it as much as possible. + +## Get Started with Buildpacks + +If you want to learn more about buildbacks, check their documentation: +https://buildpacks.io/docs/ diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8bbd34e6..04780c67 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -50,6 +50,8 @@ nav: - Remote: remote.md - Development: - Testing: development/testing.md + - Design Decisions: + - Buildpacks: design-decisions/buildpacks.md repo_url: https://github.com/DSD-DBS/capella-dockerimages edit_uri: edit/master/docs/docs diff --git a/eclipse/set_memory_flags.py b/eclipse/set_memory_flags.py old mode 100644 new mode 100755 index cfdd5406..1f6fc4ca --- a/eclipse/set_memory_flags.py +++ b/eclipse/set_memory_flags.py @@ -1,3 +1,5 @@ +#!/usr/bin/python + # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 """Set the memory flags in the *.ini file for Eclipse applications. @@ -80,8 +82,8 @@ def append_flag_to_file( if __name__ == "__main__": eclipse_executable = pathlib.Path(os.environ["ECLIPSE_EXECUTABLE"]) - _memory_min = os.environ["MEMORY_MIN"].strip() - _memory_max = os.environ["MEMORY_MAX"].strip() + _memory_min = os.environ.get("MEMORY_MIN", "70%").strip() + _memory_max = os.environ.get("MEMORY_MAX", "90%").strip() ini_path = eclipse_executable.with_suffix(".ini") diff --git a/remote/Dockerfile b/remote/Dockerfile deleted file mode 100644 index 36ba9019..00000000 --- a/remote/Dockerfile +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -ARG BASE_IMAGE=capella/base -FROM ${BASE_IMAGE} - -# Port if xrdp is used as connection method. -EXPOSE 3389 - -# Port if xpra is used as connection method. -EXPOSE 10000 - -ARG DEBIAN_FRONTEND=noninteractive - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -ENV SHELL=/bin/bash - -# Install RDP (XRDP with XORG) -USER root - -# Install xrdp and dependencies -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - xrdp \ - xserver-xorg-core \ - xorgxrdp \ - openbox \ - obconf \ - gettext-base \ - xprintidle \ - nitrogen && rm -rf /var/lib/apt/lists/* - -ARG XPRA_REGISTRY=https://xpra.org - -# Install xpra and dependencies -RUN wget -qO /usr/share/keyrings/xpra.asc ${XPRA_REGISTRY}/xpra.asc && \ - wget -qO /etc/apt/sources.list.d/xpra.sources https://raw.githubusercontent.com/Xpra-org/xpra/master/packaging/repos/bookworm/xpra.sources && \ - sed -i "s|https://xpra.org|${XPRA_REGISTRY}|" /etc/apt/sources.list.d/xpra.sources && \ - apt-get update && \ - apt-get install -y \ - --no-install-recommends \ - xpra \ - xpra-x11 \ - xpra-html5=10.1-r10-1 \ - apache2-utils \ - nginx && \ - rm -rf /var/lib/apt/lists/* - -COPY rc.xml /etc/xdg/openbox/rc.xml -COPY menu.xml /etc/xdg/openbox/menu.xml - -# Setup Nitrogen (Desktop background) -COPY wallpaper.png /tmp/wallpaper.png -COPY bg-saved.cfg /home/techuser/.config/nitrogen/bg-saved.cfg - -# Copy Supervisor Configuration -RUN pip install --no-cache-dir supervisor==4.2.5 -COPY supervisord.conf /etc/supervisord.conf -COPY supervisord.*.conf /tmp/supervisord/ - -# Copy nginx configuration for xpra -COPY nginx.conf /etc/nginx/nginx.conf -COPY error.html /usr/share/nginx/html/error.html - -# Allow any user to start the RDP server -# Depending on the base image used, Xwrapper.config may (not) be available and has to be created. -RUN sed -i 's/allowed_users=console/allowed_users=anybody/g' /etc/X11/Xwrapper.config \ - || echo "allowed_users=anybody" > /etc/X11/Xwrapper.config && \ - chmod 666 /etc/shadow - -# Set permissions -RUN mkdir -p /run/xrdp/sockdir && \ - chown -R techuser /etc/xrdp /run/xrdp /var/log/xrdp* && \ - chown techuser /var/log && \ - chown techuser /etc/supervisord.conf /var/log/nginx /var/log/nginx/* && \ - chown techuser /etc/nginx && \ - touch /etc/environment && chown techuser /etc/environment - -WORKDIR /home/techuser - -COPY startup.sh .startup.sh -RUN chmod 755 .startup.sh /home/techuser/.config/openbox/autostart - -# Prepare idletime metric endpoint -RUN pip install --no-cache-dir prometheus-client==0.17.1 - -COPY metrics.py .metrics.py -RUN chown techuser /home/techuser/.metrics.py - -USER techuser -ENTRYPOINT [ "/home/techuser/.startup.sh" ] diff --git a/remote/startup.sh b/remote/startup.sh deleted file mode 100755 index b5cb75d8..00000000 --- a/remote/startup.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -set -exuo pipefail - -RESOLVED_CONECTION_METHOD=${CONNECTION_METHOD:-xrdp} - -if [ "$RESOLVED_CONECTION_METHOD" == "xrdp" ]; -then - if [ "$(whoami)" == "root" ] || [ "$(whoami)" == "techuser" ]; - then - salt=$(openssl rand -base64 16) - password_hash=$(openssl passwd -6 -salt ${salt} "${RMT_PASSWORD:?}") - line=$(grep techuser /etc/shadow); - echo ${line%%:*}:${password_hash}:${line#*:*:} > /etc/shadow; - else - echo "Only techuser and root are supported as users."; - exit 1; - fi -fi - -# Replace Variables in the nginx.conf -sed -i "s|__XPRA_SUBPATH__|${XPRA_SUBPATH:-/}|g" /etc/nginx/nginx.conf -sed -i "s|__XPRA_CSP_ORIGIN_HOST__|${XPRA_CSP_ORIGIN_HOST:-}|g" /etc/nginx/nginx.conf - -unset RMT_PASSWORD - -# Run preparation scripts -for filename in /opt/setup/*.py; do - [ -e "$filename" ] || continue - echo "Executing Python script '$filename'..." - python3 $filename -done - -for filename in /opt/setup/*.sh; do - [ -e "$filename" ] || continue - echo "Executing shell script '$filename'..." - source $filename -done - -# Load supervisord configuration for connection method -SUPERVISORD_CONFIG_PATH=/tmp/supervisord/supervisord.${RESOLVED_CONECTION_METHOD}.conf -if [ -f "$SUPERVISORD_CONFIG_PATH" ]; then - echo "Adding '$SUPERVISORD_CONFIG_PATH' to configuration." - cat $SUPERVISORD_CONFIG_PATH >> /etc/supervisord.conf -else - echo "No '$SUPERVISORD_CONFIG_PATH' found'. Falling back to xrdp configuration." - cat /tmp/supervisord/supervisord.xrdp.conf >> /etc/supervisord.conf -fi - -echo "---START_SESSION---" - -exec supervisord diff --git a/remote/supervisord.conf b/remote/supervisord.conf deleted file mode 100644 index 48f98c59..00000000 --- a/remote/supervisord.conf +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -[program:idletime] -; Return idle time of xserver in seconds from xprintidle -command=python .metrics.py -user=techuser -autorestart=true -environment=DISPLAY=":10" - -[supervisord] -nodaemon=true -childlogdir=/var/log diff --git a/remote/tests/test_metrics.py b/remote/tests/test_metrics.py deleted file mode 100644 index 2b78a486..00000000 --- a/remote/tests/test_metrics.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 -"""Unit tests for metrics.""" -import time -import typing as t -from unittest import mock - -import metrics -import pytest - - -@pytest.fixture(name="idler") -def fixture_idler() -> metrics.IdleTimer: - """Return an instance of `metrics.IdleTime`""" - return metrics.IdleTimer() - - -def print_no_display(*__: t.Any, **_: t.Any) -> mock.MagicMock: - """ - Just print: 'could not open display', the response of xprintidle - when there is no X server. - """ - return mock.MagicMock(stdout="couldn't open display") - - -def test_get_idletime_had_no_display_on_init_but_later() -> None: - with mock.patch("subprocess.run", print_no_display): - idler = metrics.IdleTimer() - _, init_idletime = idler.first_checkpoint - - idletime = idler.get_idletime() - - assert init_idletime == -1 - assert idletime > -1 - - -def test_get_idletime_never_had_a_display_and_returns_minus_one() -> None: - with mock.patch("subprocess.run", print_no_display): - idler = metrics.IdleTimer() - _, init_idletime = idler.first_checkpoint - - time.sleep(10) - - idletime = idler.get_idletime() - - assert init_idletime == -1 - assert idletime == -1 - - -def test_get_idletime_works(idler: metrics.IdleTimer) -> None: - idletime = idler.get_idletime() - - assert isinstance(idletime, float) - - -def test_get_idletime_increases_after_display_is_closed( - idler: metrics.IdleTimer, -) -> None: - idletime = idler.get_idletime() - - time.sleep(60) - - mock.patch("subprocess.run", print_no_display) - - second_idletime = idler.get_idletime() - - assert isinstance(idletime, float) and isinstance(second_idletime, float) - assert idletime < second_idletime diff --git a/remote/wallpaper.png b/remote/wallpaper.png deleted file mode 100644 index e69de29b..00000000 diff --git a/t4c/.dockerignore.template b/t4c/.dockerignore.template deleted file mode 100644 index 59fb8993..00000000 --- a/t4c/.dockerignore.template +++ /dev/null @@ -1,7 +0,0 @@ -# Please only edit the .dockerignore.template file! -# The .dockerignore file is gitignored and updated by the Makefile -# Environment variables are replaced before build with `envsubst` - -.dockerignore.template -updateSite/* -!updateSite/$CAPELLA_VERSION diff --git a/t4c/.dockerignore.template.license b/t4c/.dockerignore.template.license deleted file mode 100644 index 7ea22469..00000000 --- a/t4c/.dockerignore.template.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -SPDX-License-Identifier: Apache-2.0 diff --git a/t4c/Dockerfile b/t4c/Dockerfile deleted file mode 100644 index 5ed300c5..00000000 --- a/t4c/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -# syntax=docker/dockerfile:1.4 - -# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors -# SPDX-License-Identifier: Apache-2.0 - -ARG BASE_IMAGE=capella/base -FROM ${BASE_IMAGE} - -ARG CAPELLA_VERSION="" -ENV CAPELLA_VERSION=${CAPELLA_VERSION} - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -ENV SHELL=/bin/bash - -USER root - -RUN apt-get update && \ - apt-get install -y git-lfs xvfb xauth && \ - rm -rf /var/lib/apt/lists/* - -COPY docker_entrypoint.sh /docker_entrypoint.sh -RUN chmod 555 /docker_entrypoint.sh - -ENV DISPLAY :99 - -WORKDIR /opt/capella - -RUN chown -R techuser /opt/capella - -# Install T4C -COPY ./updateSite/$CAPELLA_VERSION /opt/updateSite -WORKDIR /opt/updateSite -RUN find /opt/updateSite -type f -name "*.zip" -exec chmod +r {} \; - -USER techuser -## Install T4C Plugins via the P2 API from Eclipse -RUN T4C_ZIP=$(find . -type f -iname "com.thalesgroup.mde.melody.team.license.update-*.zip" | head -n 1 | cut -c 3-); \ - /opt/capella/capella \ - -consoleLog \ - -application org.eclipse.equinox.p2.director \ - -noSplash \ - -repository jar:file:///opt/updateSite/$T4C_ZIP!/ \ - -installIU com.thalesgroup.mde.melody.collab.feature.feature.group,com.thalesgroup.mde.melody.collab.maintenance.feature.feature.group,com.thalesgroup.mde.melody.collab.licbranding.feature.feature.group && \ - chown -R techuser /opt/capella/configuration - -RUN PATCH_DIR=/opt/updateSite /opt/patch.sh - -USER root - -COPY setup_workspace_t4c.py /opt/setup/setup_workspace_t4c.py - -RUN cat >> /opt/capella/capella.ini <