diff --git a/Makefile b/Makefile index 049a28b9b6..de7db6d269 100644 --- a/Makefile +++ b/Makefile @@ -256,7 +256,7 @@ synchronize-rsa-keys: -n $(NAMESPACE) \ --container $(RELEASE)-backend \ $$POD_NAME \ - -- python -m capellacollab.cli keys export /tmp/private.key + -- /opt/backend/.venv/bin/python -m capellacollab.cli keys export /tmp/private.key kubectl cp \ --context k3d-$(CLUSTER_NAME) \ diff --git a/backend/capellacollab/configuration/app/models.py b/backend/capellacollab/configuration/app/models.py index c9f5188986..6773d7939b 100644 --- a/backend/capellacollab/configuration/app/models.py +++ b/backend/capellacollab/configuration/app/models.py @@ -118,7 +118,7 @@ class K8sPromtailConfig(BaseConfig): examples=[True], ) loki_url: str | None = pydantic.Field( - default="http://localhost:30001/loki/api/v1/push", + default="http://dev-loki-gateway.collab-manager.svc.cluster.local/loki/api/v1", alias="lokiURL", description="The URL of the Loki instance to which to push logs.", examples=["http://localhost:30001/loki/api/v1/push"], diff --git a/backend/capellacollab/sessions/hooks/jupyter.py b/backend/capellacollab/sessions/hooks/jupyter.py index d2e2c99764..d1bb71106f 100644 --- a/backend/capellacollab/sessions/hooks/jupyter.py +++ b/backend/capellacollab/sessions/hooks/jupyter.py @@ -102,6 +102,7 @@ def _get_project_share_volume_mounts( / model.project.slug / model.slug, volume_name=volume_name, + sub_path=None, ) ) diff --git a/backend/capellacollab/sessions/hooks/log_collector.py b/backend/capellacollab/sessions/hooks/log_collector.py index b8e8e76716..978c2a5ba1 100644 --- a/backend/capellacollab/sessions/hooks/log_collector.py +++ b/backend/capellacollab/sessions/hooks/log_collector.py @@ -10,7 +10,6 @@ from capellacollab.sessions import models as sessions_models from capellacollab.sessions.operators import models as operators_models from capellacollab.tools import models as tools_models -from capellacollab.users.workspaces import crud as users_workspaces_crud from . import interface @@ -20,6 +19,23 @@ class LogCollectorIntegration(interface.HookRegistration): _loki_enabled: bool = config.k8s.promtail.loki_enabled + def configuration_hook(self, request: interface.ConfigurationHookRequest): + return interface.ConfigurationHookResult( + volumes=[self._get_logs_volume(session_id=request.session_id)] + ) + + @classmethod + def _get_logs_volume( + cls, session_id: str + ) -> operators_models.PersistentVolume: + return operators_models.PersistentVolume( + name="logs", + read_only=False, + container_path=pathlib.PurePosixPath("/var/log/session"), + volume_name=f"{config.k8s.release_name}-session-logs", + sub_path=session_id, + ) + def post_session_creation_hook( self, request: interface.PostSessionCreationHookRequest, @@ -27,12 +43,6 @@ def post_session_creation_hook( if not self._log_collection_enabled(request.db_session): return interface.PostSessionCreationHookResult() - workspaces = users_workspaces_crud.get_workspaces_for_user( - request.db, request.user - ) - if not workspaces: - return interface.PostSessionCreationHookResult() - request.operator._create_configmap( name=request.db_session.id, data=self._promtail_configuration( @@ -58,13 +68,9 @@ def post_session_creation_hook( container_path=pathlib.PurePosixPath("/etc/promtail"), config_map_name=request.db_session.id, optional=False, + sub_path=None, ), - operators_models.PersistentVolume( - name="workspace", - read_only=False, - container_path=pathlib.PurePosixPath("/workspace"), - volume_name=workspaces[0].pvc_name, - ), + self._get_logs_volume(session_id=request.db_session.id), ] request.operator._create_sidecar_pod( @@ -97,7 +103,6 @@ def _log_collection_enabled( ) -> bool: return ( self._loki_enabled - and session.type == sessions_models.SessionType.PERSISTENT and session.tool.config.monitoring.logging.enabled ) @@ -129,7 +134,7 @@ def _promtail_configuration( } ], "positions": { - "filename": f"/workspace/.promtail/positions-tool-{tool.id}.yaml" + "filename": "/var/log/session/.positions.yaml" }, "scrape_configs": [ { @@ -143,7 +148,6 @@ def _promtail_configuration( ], "static_configs": [ { - "targets": ["localhost"], "labels": { "username": username, "session_type": session_type, @@ -151,7 +155,7 @@ def _promtail_configuration( "tool_id": tool.id, "version_id": version.id, "connection_method_id": connection_method.id, - "__path__": tool.config.monitoring.logging.path, + "__path__": "/var/log/session/**/*.log", }, } ], diff --git a/backend/capellacollab/sessions/hooks/persistent_workspace.py b/backend/capellacollab/sessions/hooks/persistent_workspace.py index 98e7ab37ed..23693d37a0 100644 --- a/backend/capellacollab/sessions/hooks/persistent_workspace.py +++ b/backend/capellacollab/sessions/hooks/persistent_workspace.py @@ -42,6 +42,7 @@ def configuration_hook( read_only=False, container_path=pathlib.PurePosixPath("/workspace"), volume_name=volume_name, + sub_path=None, ) return interface.ConfigurationHookResult( diff --git a/backend/capellacollab/sessions/hooks/pure_variants.py b/backend/capellacollab/sessions/hooks/pure_variants.py index f0016ee620..a6038012ca 100644 --- a/backend/capellacollab/sessions/hooks/pure_variants.py +++ b/backend/capellacollab/sessions/hooks/pure_variants.py @@ -82,6 +82,7 @@ def configuration_hook( container_path=pathlib.PurePosixPath("/inputs/pure-variants"), secret_name="pure-variants", optional=True, + sub_path=None, ) return interface.ConfigurationHookResult( diff --git a/backend/capellacollab/sessions/hooks/read_only_workspace.py b/backend/capellacollab/sessions/hooks/read_only_workspace.py index cbe1b45119..5de0553812 100644 --- a/backend/capellacollab/sessions/hooks/read_only_workspace.py +++ b/backend/capellacollab/sessions/hooks/read_only_workspace.py @@ -25,6 +25,7 @@ def configuration_hook( name="workspace", read_only=False, container_path=pathlib.PurePosixPath("/workspace"), + sub_path=None, ) ], ) diff --git a/backend/capellacollab/sessions/hooks/session_preparation.py b/backend/capellacollab/sessions/hooks/session_preparation.py index e81c7f53a7..88dc5d0efa 100644 --- a/backend/capellacollab/sessions/hooks/session_preparation.py +++ b/backend/capellacollab/sessions/hooks/session_preparation.py @@ -28,6 +28,7 @@ def configuration_hook( request.tool.config.provisioning.directory ), read_only=False, + sub_path=None, ) return interface.ConfigurationHookResult( diff --git a/backend/capellacollab/sessions/operators/k8s.py b/backend/capellacollab/sessions/operators/k8s.py index eb2797238d..1c181ee520 100644 --- a/backend/capellacollab/sessions/operators/k8s.py +++ b/backend/capellacollab/sessions/operators/k8s.py @@ -419,6 +419,7 @@ def _map_volumes_to_k8s_volumes( name=volume.name, mount_path=str(volume.container_path), read_only=volume.read_only, + sub_path=volume.sub_path, ) ) diff --git a/backend/capellacollab/sessions/operators/models.py b/backend/capellacollab/sessions/operators/models.py index d58ee7ed99..b26b743fca 100644 --- a/backend/capellacollab/sessions/operators/models.py +++ b/backend/capellacollab/sessions/operators/models.py @@ -11,6 +11,7 @@ class Volume(metaclass=abc.ABCMeta): name: str read_only: bool container_path: pathlib.PurePosixPath + sub_path: str | None @dataclasses.dataclass diff --git a/backend/capellacollab/tools/models.py b/backend/capellacollab/tools/models.py index 992ef6e52e..22212ecdda 100644 --- a/backend/capellacollab/tools/models.py +++ b/backend/capellacollab/tools/models.py @@ -276,11 +276,6 @@ class LoggingConfiguration(core_pydantic.BaseModel): description="If enabled, logs will be pushed to Grafana Loki.", ) - path: str = pydantic.Field( - default="/workspace/**/*.log", - description="Path to the log files, can be a glob string.", - ) - class SessionMonitoring(core_pydantic.BaseModel): prometheus: PrometheusConfiguration = pydantic.Field( diff --git a/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py b/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py index 00c4b721ba..bcc36ac639 100644 --- a/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py +++ b/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py @@ -19,6 +19,7 @@ def test_secret_reference_volume_mapping(): container_path=pathlib.PurePosixPath("/inputs/test"), secret_name="test", optional=True, + sub_path=None, ) ] @@ -47,6 +48,7 @@ def test_persistent_volume_mapping(): read_only=True, container_path=pathlib.PurePosixPath("/inputs/test"), volume_name="volume_test", + sub_path=None, ) ] @@ -73,6 +75,7 @@ def test_empty_volume_mapping(): name="test", read_only=True, container_path=pathlib.PurePosixPath("/inputs/test"), + sub_path=None, ) ] diff --git a/helm/config/grafana/dashboards/individual-session.json b/helm/config/grafana/dashboards/individual-session.json index 15cf813b4e..7020c3fba5 100644 --- a/helm/config/grafana/dashboards/individual-session.json +++ b/helm/config/grafana/dashboards/individual-session.json @@ -25,9 +25,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 15, + "id": 11, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -78,7 +77,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -145,7 +144,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -212,7 +211,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -280,7 +279,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -317,6 +316,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", @@ -373,11 +373,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -413,6 +414,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -473,10 +475,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -514,6 +518,7 @@ "axisPlacement": "auto", "axisSoftMin": -1, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "scheme", @@ -578,10 +583,12 @@ "sortDesc": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -650,7 +657,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -658,7 +665,7 @@ "uid": "prometheus_ccm" }, "editorMode": "code", - "expr": "count_over_time(up{session_id=~\"$session_id\"}[$__range])/2", + "expr": "count_over_time(up{session_id=~\"$session_id\", job=\"sessions\"}[$__range])/2", "instant": false, "legendFormat": "__auto", "range": true, @@ -686,6 +693,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -747,10 +755,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -786,6 +796,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -841,11 +852,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.1.0", + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -933,6 +945,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -995,10 +1008,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -1022,6 +1037,10 @@ "uid": "loki_ccm" }, "description": "The oldest logs are displayed first.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 18, "w": 12, @@ -1031,6 +1050,7 @@ "id": 23, "options": { "dedupStrategy": "none", + "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, @@ -1039,14 +1059,16 @@ "sortOrder": "Ascending", "wrapLogMessage": false }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { "type": "loki", "uid": "loki_ccm" }, - "editorMode": "builder", - "expr": "{session_id=\"$session_id\"} |= ``", + "direction": "backward", + "editorMode": "code", + "expr": "{session_id=\"$session_id\", filename=~\"$filename\"} |= ``", "queryType": "range", "refId": "A" } @@ -1072,6 +1094,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1103,8 +1126,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1133,10 +1155,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.5.1", "targets": [ { "datasource": { @@ -1155,14 +1179,14 @@ "type": "timeseries" } ], + "preload": false, "refresh": "", - "schemaVersion": 39, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { "current": { - "selected": false, "text": "", "value": "" }, @@ -1172,10 +1196,8 @@ }, "definition": "label_values(idletime_minutes,session_id)", "description": "Unique identifier of a session", - "hide": 0, "includeAll": false, "label": "Session ID", - "multi": false, "name": "session_id", "options": [], "query": { @@ -1185,9 +1207,34 @@ }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "current": { + "text": "All", + "value": ["$__all"] + }, + "datasource": { + "type": "loki", + "uid": "loki_ccm" + }, + "definition": "", + "description": "Filename of the file in the logs", + "includeAll": true, + "label": "Filename of logs", + "multi": true, + "name": "filename", + "options": [], + "query": { + "label": "filename", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{session_id=\"$session_id\"}", + "type": 1 + }, + "refresh": 1, + "regex": "(?(\\/var\\/log\\/session\\/(?.+)))", + "type": "query" } ] }, diff --git a/helm/config/grafana/dashboards/management-portal-logs.json b/helm/config/grafana/dashboards/management-portal-logs.json index 065c3b2c25..21479a96fb 100644 --- a/helm/config/grafana/dashboards/management-portal-logs.json +++ b/helm/config/grafana/dashboards/management-portal-logs.json @@ -31,12 +31,12 @@ "overrides": [] }, "gridPos": { - "h": 29, + "h": 10, "w": 12, "x": 0, "y": 0 }, - "id": 1, + "id": 3, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, @@ -56,13 +56,13 @@ "uid": "loki_ccm" }, "direction": "backward", - "editorMode": "code", - "expr": "{deployment=~\".*-deployment-backend\"} |= `` | logfmt", + "editorMode": "builder", + "expr": "{deployment=~\".*-deployment-backend\"} |= `` | logfmt | level = `ERROR`", "queryType": "range", "refId": "A" } ], - "title": "Backend Logs", + "title": "Backend Error Logs", "type": "logs" }, { @@ -75,7 +75,7 @@ "overrides": [] }, "gridPos": { - "h": 29, + "h": 20, "w": 12, "x": 12, "y": 0 @@ -108,6 +108,50 @@ ], "title": "Frontend Logs", "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "loki_ccm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki_ccm" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{deployment=~\".*-deployment-backend\"} |= `` | logfmt", + "queryType": "range", + "refId": "A" + } + ], + "title": "Backend Logs", + "type": "logs" } ], "preload": false, @@ -125,6 +169,6 @@ "timezone": "browser", "title": "Management Portal Logs", "uid": "management-portal-logs", - "version": 3, + "version": 1, "weekStart": "" } diff --git a/helm/templates/sessions/session-logs.volume.yaml b/helm/templates/sessions/session-logs.volume.yaml new file mode 100644 index 0000000000..2801f35b0e --- /dev/null +++ b/helm/templates/sessions/session-logs.volume.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +{{ if .Values.loki.enabled }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-session-logs + namespace: {{ .Values.backend.k8sSessionNamespace }} + labels: + id: {{ .Release.Name }}-pvc-session-logs + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - {{ .Values.backend.storageAccessMode }} + resources: + requests: + storage: 10Gi + storageClassName: {{ .Values.backend.storageClassName }} +{{ end }}