feat!: Replace Jupyter file-shares with generic project level volumes

backend/capellacollab/alembic/versions/b2fd698ed6cb_add_general_file_shares.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add general file-shares
+
+Revision ID: b2fd698ed6cb
+Revises: ca9ce61491a7
+Create Date: 2025-02-21 15:41:21.391386
+
+"""
+
+import datetime
+import logging
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+logger = logging.getLogger(__name__)
+
+# revision identifiers, used by Alembic.
+revision = "b2fd698ed6cb"
+down_revision = "ca9ce61491a7"
+branch_labels = None
+depends_on = None
+
+t_tool_models = sa.Table(
+    "tool_models",
+    sa.MetaData(),
+    sa.Column("id", sa.Integer(), primary_key=True),
+    sa.Column("project_id", sa.Integer()),
+    sa.Column("tool_id"),
+    sa.Column("configuration", postgresql.JSONB(astext_type=sa.Text())),
+)
+
+t_tools = sa.Table(
+    "tools",
+    sa.MetaData(),
+    sa.Column("id", sa.Integer()),
+    sa.Column("integrations", postgresql.JSONB(astext_type=sa.Text())),
+)
+
+
+def upgrade():
+    t_project_volumes = op.create_table(
+        "project_volumes",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("pvc_name", sa.String(), nullable=False),
+        sa.Column("size", sa.String(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projects.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("pvc_name"),
+    )
+    op.create_index(
+        op.f("ix_project_volumes_id"), "project_volumes", ["id"], unique=False
+    )
+
+    connection = op.get_bind()
+    tools = connection.execute(sa.select(t_tools)).mappings().all()
+
+    jupyter_tool_ids = []
+    for tool in tools:
+        if tool["integrations"]["jupyter"]:
+            jupyter_tool_ids.append(tool["id"])
+
+    tool_models = (
+        connection.execute(
+            sa.select(t_tool_models).where(
+                t_tool_models.c.tool_id.in_(jupyter_tool_ids)
+            )
+        )
+        .mappings()
+        .all()
+    )
+
+    grouped_by_project = {}
+    for tool_model in tool_models:
+        logger.info(
+            "Adding general file share for Jupyter model %s in project %s",
+            tool_model["id"],
+            tool_model["project_id"],
+        )
+
+        project_id = tool_model["project_id"]
+        if project_id in grouped_by_project:
+            raise ValueError(
+                "Due a removal of the abstraction layer for"
+                " Jupyter file-shares, it's no longer possible to"
+                " have multiple file-shares in one project."
+                f" In the project {project_id} we've identified multiple Jupyter models."
+                " Please rollback, merge the files in the file-shares manually"
+                " and remove empty Jupyter models."
+                " Then try again."
+            )
+
+        if "workspace" in tool_model["configuration"]:
+            grouped_by_project[project_id] = {
+                "created_at": datetime.datetime.now(tz=datetime.UTC),
+                "pvc_name": "shared-workspace-"
+                + tool_model["configuration"]["workspace"],
+                "project_id": tool_model["project_id"],
+                "size": "2Gi",
+            }
+
+    op.bulk_insert(
+        t_project_volumes,
+        list(grouped_by_project.values()),
+    )

backend/capellacollab/core/database/models.py

@@ -19,6 +19,7 @@
 import capellacollab.projects.toolmodels.restrictions.models
 import capellacollab.projects.tools.models
 import capellacollab.projects.users.models
+import capellacollab.projects.volumes.models
 import capellacollab.sessions.models
 import capellacollab.settings.integrations.purevariants.models
 import capellacollab.settings.modelsources.git.models

backend/capellacollab/projects/permissions/models.py

@@ -168,14 +168,18 @@ class ProjectUserScopes(core_pydantic.BaseModel):
         t.Literal[
             permissions_models.UserTokenVerb.GET,
             permissions_models.UserTokenVerb.UPDATE,
+            permissions_models.UserTokenVerb.CREATE,
+            permissions_models.UserTokenVerb.DELETE,
         ]
     ] = pydantic.Field(
         default_factory=set,
         title="Shared Workspaces",
         description=(
-            "Access to shared Jupyter workspaces."
+            "Access to shared project volumes / workspaces."
             " GET will provide read-only access,"
-            " UPDATE will provide read & write access."
+            " UPDATE will provide read & write access,"
+            " CREATE will provide the ability to create a new shared volume,"
+            " DELETE will provide the ability to delete shared volumes."
         ),
     )
 

backend/capellacollab/projects/routes.py

@@ -32,6 +32,7 @@
 from capellacollab.projects.users import crud as projects_users_crud
 from capellacollab.projects.users import models as projects_users_models
 from capellacollab.projects.users import routes as projects_users_routes
+from capellacollab.projects.volumes import routes as volumes_routes
 from capellacollab.users import injectables as users_injectables
 from capellacollab.users import models as users_models
 from capellacollab.users.tokens import models as tokens_models
@@ -283,3 +284,7 @@ def _delete_all_pipelines_for_project(
     prefix="/-/permissions",
     tags=["Projects - Permissions"],
 )
+router.include_router(
+    volumes_routes.router,
+    prefix="/{project_slug}",
+)

backend/capellacollab/projects/toolmodels/crud.py

@@ -82,7 +82,6 @@ def create_model(
     tool: tools_models.DatabaseTool,
     version: tools_models.DatabaseVersion | None = None,
     nature: tools_models.DatabaseNature | None = None,
-    configuration: dict[str, str] | None = None,
     display_order: int | None = None,
 ) -> models.DatabaseToolModel:
     model = models.DatabaseToolModel(
@@ -93,7 +92,6 @@ def create_model(
         tool=tool,
         version=version,
         nature=nature,
-        configuration=configuration,
         display_order=display_order,
     )
 

backend/capellacollab/projects/toolmodels/routes.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import typing as t
-import uuid
 
 import fastapi
 import slugify
@@ -26,7 +25,7 @@
 from capellacollab.users import models as users_models
 from capellacollab.users.tokens import models as tokens_models
 
-from . import crud, exceptions, injectables, models, workspace
+from . import crud, exceptions, injectables, models
 from .backups import routes as backups_routes
 from .diagrams import routes as diagrams_routes
 from .modelbadge import routes as complexity_badge_routes
@@ -109,26 +108,14 @@ def create_new_tool_model(
     tool = tools_injectables.get_existing_tool(
         tool_id=new_model.tool_id, db=db
     )
-    configuration = {}
-    if tool.integrations.jupyter:
-        configuration["workspace"] = str(uuid.uuid4())
 
     slug = slugify.slugify(new_model.name)
     if project.type not in tool.config.supported_project_types:
         raise exceptions.ProjectTypeNotSupportedByToolModel(project.slug, slug)
     if crud.get_model_by_slugs(db, project.slug, slug):
         raise exceptions.ToolModelAlreadyExistsError(project.slug, slug)
 
-    model = crud.create_model(
-        db, project, new_model, tool, configuration=configuration
-    )
-
-    if tool.integrations.jupyter:
-        workspace.create_shared_workspace(
-            configuration["workspace"], project, model, "2Gi"
-        )
-
-    return model
+    return crud.create_model(db, project, new_model, tool)
 
 
 @router.patch(
@@ -269,13 +256,6 @@ def delete_tool_model(
             model.name, f"{model.tool.name} model", dependencies
         )
 
-    if (
-        model.tool.integrations.jupyter
-        and model.configuration
-        and "workspace" in model.configuration
-    ):
-        workspace.delete_shared_workspace(model.configuration["workspace"])
-
     crud.delete_model(db, model)
 
 

backend/capellacollab/projects/volumes/crud.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0

backend/capellacollab/projects/volumes/exceptions.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from fastapi import status
+
+from capellacollab.core import exceptions
+
+
+class OnlyOneVolumePerProjectError(exceptions.BaseError):
+    def __init__(self, project_slug: str):
+        super().__init__(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            title="One one volume per project allowed",
+            reason=(
+                f"You can't add another volume to the project '{project_slug}'."
+                " Each project can only have a maximum of one volume."
+                " Reuse the existing volume or delete the old volume first."
+            ),
+            err_code="ONLY_ONE_VOLUME_PER_PROJECT",
+        )
+
+    @classmethod
+    def openapi_example(cls) -> "OnlyOneVolumePerProjectError":
+        return cls("test")
+
+
+class VolumeNotFoundError(exceptions.BaseError):
+    def __init__(self, volume_id: int, project_slug: str):
+        super().__init__(
+            status_code=status.HTTP_404_NOT_FOUND,
+            title="Project volume not found",
+            reason=(
+                f"Couldn't find the volume {volume_id} in the project '{project_slug}'."
+            ),
+            err_code="PROJECT_VOLUME_NOT_FOUND",
+        )
+
+    @classmethod
+    def openapi_example(cls) -> "VolumeNotFoundError":
+        return cls(-1, "test")

backend/capellacollab/projects/volumes/models.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+import datetime
+
+import pydantic
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from capellacollab.core import database
+from capellacollab.core import pydantic as core_pydantic
+from capellacollab.projects import models as projects_models
+
+
+class ProjectVolume(core_pydantic.BaseModel):
+    created_at: datetime.datetime
+    size: str
+    pvc_name: str
+
+    _validate_created_at = pydantic.field_serializer("created_at")(
+        core_pydantic.datetime_serializer
+    )
+
+
+class DatabaseProjectVolume(database.Base):
+    __tablename__ = "project_volumes"
+
+    id: orm.Mapped[int] = orm.mapped_column(
+        init=False, primary_key=True, index=True, autoincrement=True
+    )
+
+    created_at: orm.Mapped[datetime.datetime]
+    pvc_name: orm.Mapped[str] = orm.mapped_column(unique=True)
+    size: orm.Mapped[str]
+
+    project_id: orm.Mapped[int] = orm.mapped_column(
+        sa.ForeignKey("projects.id"), init=False
+    )
+    project: orm.Mapped[projects_models.DatabaseProject] = orm.relationship()

backend/capellacollab/projects/volumes/routes.py

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+import fastapi
+
+from capellacollab.permissions import models as permissions_models
+from capellacollab.projects.permissions import (
+    injectables as projects_permissions_injectables,
+)
+from capellacollab.projects.permissions import (
+    models as projects_permissions_models,
+)
+
+from . import models
+
+router = fastapi.APIRouter(
+    prefix="/volumes",
+    tags=["Projects - Volumes"],
+)
+
+
+@router.get(
+    "",
+    dependencies=[
+        fastapi.Depends(
+            projects_permissions_injectables.ProjectPermissionValidation(
+                required_scope=projects_permissions_models.ProjectUserScopes(
+                    project_users={permissions_models.UserTokenVerb.GET}
+                )
+            )
+        )
+    ],
+)
+def get_project_volumes() -> models.ProjectVolume:
+    return []
+
+
+@router.get(
+    "",
+    dependencies=[
+        fastapi.Depends(
+            projects_permissions_injectables.ProjectPermissionValidation(
+                required_scope=projects_permissions_models.ProjectUserScopes(
+                    project_users={permissions_models.UserTokenVerb.CREATE}
+                )
+            )
+        )
+    ],
+)
+def create_project_volume():
+    return {}
+
+
+@router.get(
+    "",
+    dependencies=[
+        fastapi.Depends(
+            projects_permissions_injectables.ProjectPermissionValidation(
+                required_scope=projects_permissions_models.ProjectUserScopes(
+                    project_users={permissions_models.UserTokenVerb.DELETE}
+                )
+            )
+        )
+    ],
+)
+def delete_project_volume():
+    return {}