feat!: Collect logs per session individually

Logs are collected from `/var/log/session` instead of the custom paths. This enabled better separation between logs for specific sessions. Also enables support for logs in read-only sessions. Signed-off-by: MoritzWeber <[email protected]>
DSD-DBS · Feb 20, 2025 · 858340d · 858340d
1 parent d86b97b
commit 858340d
Show file tree

Hide file tree

Showing 18 changed files with 171 additions and 59 deletions.
diff --git a/Makefile b/Makefile
@@ -256,7 +256,7 @@ synchronize-rsa-keys:
 		-n $(NAMESPACE) \
 		--container $(RELEASE)-backend \
 		$$POD_NAME \
-		-- python -m capellacollab.cli keys export /tmp/private.key
+		-- /opt/backend/.venv/bin/python -m capellacollab.cli keys export /tmp/private.key
 
 	kubectl cp \
 		--context k3d-$(CLUSTER_NAME) \

diff --git a/backend/capellacollab/configuration/app/models.py b/backend/capellacollab/configuration/app/models.py
@@ -118,7 +118,7 @@ class K8sPromtailConfig(BaseConfig):
         examples=[True],
     )
     loki_url: str | None = pydantic.Field(
-        default="http://localhost:30001/loki/api/v1/push",
+        default="http://dev-loki-gateway.collab-manager.svc.cluster.local/loki/api/v1",
         alias="lokiURL",
         description="The URL of the Loki instance to which to push logs.",
         examples=["http://localhost:30001/loki/api/v1/push"],

diff --git a/backend/capellacollab/sessions/hooks/jupyter.py b/backend/capellacollab/sessions/hooks/jupyter.py
@@ -102,6 +102,7 @@ def _get_project_share_volume_mounts(
                     / model.project.slug
                     / model.slug,
                     volume_name=volume_name,
+                    sub_path=None,
                 )
             )
 

diff --git a/backend/capellacollab/sessions/hooks/log_collector.py b/backend/capellacollab/sessions/hooks/log_collector.py
@@ -10,7 +10,6 @@
 from capellacollab.sessions import models as sessions_models
 from capellacollab.sessions.operators import models as operators_models
 from capellacollab.tools import models as tools_models
-from capellacollab.users.workspaces import crud as users_workspaces_crud
 
 from . import interface
 
@@ -20,19 +19,30 @@
 class LogCollectorIntegration(interface.HookRegistration):
     _loki_enabled: bool = config.k8s.promtail.loki_enabled
 
+    def configuration_hook(self, request: interface.ConfigurationHookRequest):
+        return interface.ConfigurationHookResult(
+            volumes=[self._get_logs_volume(session_id=request.session_id)]
+        )
+
+    @classmethod
+    def _get_logs_volume(
+        cls, session_id: str
+    ) -> operators_models.PersistentVolume:
+        return operators_models.PersistentVolume(
+            name="logs",
+            read_only=False,
+            container_path=pathlib.PurePosixPath("/var/log/session"),
+            volume_name=f"{config.k8s.release_name}-session-logs",
+            sub_path=session_id,
+        )
+
     def post_session_creation_hook(
         self,
         request: interface.PostSessionCreationHookRequest,
     ) -> interface.PostSessionCreationHookResult:
         if not self._log_collection_enabled(request.db_session):
             return interface.PostSessionCreationHookResult()
 
-        workspaces = users_workspaces_crud.get_workspaces_for_user(
-            request.db, request.user
-        )
-        if not workspaces:
-            return interface.PostSessionCreationHookResult()
-
         request.operator._create_configmap(
             name=request.db_session.id,
             data=self._promtail_configuration(
@@ -58,13 +68,9 @@ def post_session_creation_hook(
                 container_path=pathlib.PurePosixPath("/etc/promtail"),
                 config_map_name=request.db_session.id,
                 optional=False,
+                sub_path=None,
             ),
-            operators_models.PersistentVolume(
-                name="workspace",
-                read_only=False,
-                container_path=pathlib.PurePosixPath("/workspace"),
-                volume_name=workspaces[0].pvc_name,
-            ),
+            self._get_logs_volume(session_id=request.db_session.id),
         ]
 
         request.operator._create_sidecar_pod(
@@ -97,7 +103,6 @@ def _log_collection_enabled(
     ) -> bool:
         return (
             self._loki_enabled
-            and session.type == sessions_models.SessionType.PERSISTENT
             and session.tool.config.monitoring.logging.enabled
         )
 
@@ -129,7 +134,7 @@ def _promtail_configuration(
                         }
                     ],
                     "positions": {
-                        "filename": f"/workspace/.promtail/positions-tool-{tool.id}.yaml"
+                        "filename": "/var/log/session/.positions.yaml"
                     },
                     "scrape_configs": [
                         {
@@ -143,15 +148,14 @@ def _promtail_configuration(
                             ],
                             "static_configs": [
                                 {
-                                    "targets": ["localhost"],
                                     "labels": {
                                         "username": username,
                                         "session_type": session_type,
                                         "session_id": session_id,
                                         "tool_id": tool.id,
                                         "version_id": version.id,
                                         "connection_method_id": connection_method.id,
-                                        "__path__": tool.config.monitoring.logging.path,
+                                        "__path__": "/var/log/session/**/*.log",
                                     },
                                 }
                             ],

diff --git a/backend/capellacollab/sessions/hooks/persistent_workspace.py b/backend/capellacollab/sessions/hooks/persistent_workspace.py
@@ -42,6 +42,7 @@ def configuration_hook(
             read_only=False,
             container_path=pathlib.PurePosixPath("/workspace"),
             volume_name=volume_name,
+            sub_path=None,
         )
 
         return interface.ConfigurationHookResult(

diff --git a/backend/capellacollab/sessions/hooks/pure_variants.py b/backend/capellacollab/sessions/hooks/pure_variants.py
@@ -82,6 +82,7 @@ def configuration_hook(
             container_path=pathlib.PurePosixPath("/inputs/pure-variants"),
             secret_name="pure-variants",
             optional=True,
+            sub_path=None,
         )
 
         return interface.ConfigurationHookResult(

diff --git a/backend/capellacollab/sessions/hooks/read_only_workspace.py b/backend/capellacollab/sessions/hooks/read_only_workspace.py
@@ -25,6 +25,7 @@ def configuration_hook(
                     name="workspace",
                     read_only=False,
                     container_path=pathlib.PurePosixPath("/workspace"),
+                    sub_path=None,
                 )
             ],
         )
diff --git a/backend/capellacollab/sessions/hooks/session_preparation.py b/backend/capellacollab/sessions/hooks/session_preparation.py
@@ -28,6 +28,7 @@ def configuration_hook(
                 request.tool.config.provisioning.directory
             ),
             read_only=False,
+            sub_path=None,
         )
 
         return interface.ConfigurationHookResult(

diff --git a/backend/capellacollab/sessions/operators/k8s.py b/backend/capellacollab/sessions/operators/k8s.py
@@ -419,6 +419,7 @@ def _map_volumes_to_k8s_volumes(
                     name=volume.name,
                     mount_path=str(volume.container_path),
                     read_only=volume.read_only,
+                    sub_path=volume.sub_path,
                 )
             )
 

diff --git a/backend/capellacollab/sessions/operators/models.py b/backend/capellacollab/sessions/operators/models.py
@@ -11,6 +11,7 @@ class Volume(metaclass=abc.ABCMeta):
     name: str
     read_only: bool
     container_path: pathlib.PurePosixPath
+    sub_path: str | None
 
 
 @dataclasses.dataclass

diff --git a/backend/capellacollab/tools/models.py b/backend/capellacollab/tools/models.py
@@ -276,11 +276,6 @@ class LoggingConfiguration(core_pydantic.BaseModel):
         description="If enabled, logs will be pushed to Grafana Loki.",
     )
 
-    path: str = pydantic.Field(
-        default="/workspace/**/*.log",
-        description="Path to the log files, can be a glob string.",
-    )
-
 
 class SessionMonitoring(core_pydantic.BaseModel):
     prometheus: PrometheusConfiguration = pydantic.Field(

diff --git a/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py b/backend/tests/sessions/k8s_operator/test_session_volume_mapping.py
@@ -19,6 +19,7 @@ def test_secret_reference_volume_mapping():
             container_path=pathlib.PurePosixPath("/inputs/test"),
             secret_name="test",
             optional=True,
+            sub_path=None,
         )
     ]
 
@@ -47,6 +48,7 @@ def test_persistent_volume_mapping():
             read_only=True,
             container_path=pathlib.PurePosixPath("/inputs/test"),
             volume_name="volume_test",
+            sub_path=None,
         )
     ]
 
@@ -73,6 +75,7 @@ def test_empty_volume_mapping():
             name="test",
             read_only=True,
             container_path=pathlib.PurePosixPath("/inputs/test"),
+            sub_path=None,
         )
     ]
 

diff --git a/frontend/src/app/openapi/model/logging-configuration-input.ts b/frontend/src/app/openapi/model/logging-configuration-input.ts
diff --git a/frontend/src/app/openapi/model/logging-configuration-output.ts b/frontend/src/app/openapi/model/logging-configuration-output.ts
diff --git a/frontend/src/storybook/tool.ts b/frontend/src/storybook/tool.ts
@@ -93,7 +93,6 @@ const defaultToolConfig: ToolSessionConfigurationOutput = {
     },
     logging: {
       enabled: true,
-      path: '/workspace/*.log',
     },
   },
   supported_project_types: ['general', 'training'],