[WIP] Store files metadata in database

.env.example

@@ -38,17 +38,38 @@ LETSENCRYPT_RSA_KEY_SIZE=4096
 # Set to 1 if you're testing your setup to avoid hitting request limits
 LETSENCRYPT_STAGING=1
 
-STORAGE_ACCESS_KEY_ID=minioadmin
-STORAGE_SECRET_ACCESS_KEY=minioadmin
-STORAGE_BUCKET_NAME=qfieldcloud-local
-STORAGE_REGION_NAME=
-
-# URL to the storage endpoint either minio, or external (e.g. S3).
-# The URL must be reachable both from within docker and from the host, the default value is the `bridge` docker URL.
-# Read more on https://docs.docker.com/network/network-tutorial-standalone/ .
-# NOTE: to use minio on windows/mac, change the value to "http://host.docker.internal:8009"
-# DEFAULT: http://172.17.0.1:8009
-STORAGE_ENDPOINT_URL=http://172.17.0.1:8009
+
+# Used to define storages in QFieldCloud. Note the contents of this variable is a superset of Django's `STORAGES` setting.
+# NOTE: Note if the DYNAMIC_STORAGES is not available, QFieldCloud will still work with `STORAGE_ACCESS_KEY_ID`, `STORAGE_SECRET_KEY_ID`, `STORAGE_BUCKET_NAME` and `STORAGE_REGION_NAME` from previous QFC versions.
+# NOTE: The custom property `QFC_IS_LEGACY` is temporary available to allow migration from the old to the new way of handling files. This option will soon be removed, so you are highly encouraged to migrate all the projects to the new way of handling files.
+# NOTE: The `endpoint_url` must be a URL reachable from within docker and the host, the default value `172.17.0.1` for `minio` is the docker network `bridge`. On windows/mac, change the value to "http://host.docker.internal:8009".
+# DEFAULT:
+# {
+#     "default": {
+#         "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
+#         "OPTIONS": {
+#             "access_key": "minioadmin",
+#             "secret_key": "minioadmin",
+#             "bucket_name": "qfieldcloud-local",
+#             "region_name": "",
+#             "endpoint_url": "http://172.17.0.1:8009"
+#         },
+#         "QFC_IS_LEGACY": false
+#     }
+# }
+STORAGES='{
+        "default": {
+            "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
+            "OPTIONS": {
+                "access_key": "minioadmin",
+                "secret_key": "minioadmin",
+                "bucket_name": "qfieldcloud-local",
+                "region_name": "",
+                "endpoint_url": "http://172.17.0.1:8009"
+            },
+            "QFC_IS_LEGACY": false
+        }
+    }'
 
 # Public port to the minio API endpoint. It must match the configured port in `STORAGE_ENDPOINT_URL`.
 # NOTE: active only when minio is the configured as storage endpoint. Mostly for local development.

.github/workflows/test.yml

@@ -43,7 +43,11 @@ jobs:
           - notifs
           - subscription
           - core
+          - filestorage
           - __flaky__
+        storage:
+          - default
+          - legacy_storage
     continue-on-error: true
     steps:
       - name: Checkout repo
@@ -61,6 +65,35 @@ jobs:
           sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env
           eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g')
 
+          cat <<EOF >> .env
+          STORAGES='{
+              "legacy_storage": {
+                  "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
+                  "OPTIONS": {
+                      "access_key": "minioadmin",
+                      "secret_key": "minioadmin",
+                      "bucket_name": "qfieldcloud-local-legacy",
+                      "region_name": "",
+                      "endpoint_url": "http://172.17.0.1:8009"
+                  },
+                  "QFC_IS_LEGACY": true
+              },
+              "default": {
+                  "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
+                  "OPTIONS": {
+                      "access_key": "minioadmin",
+                      "secret_key": "minioadmin",
+                      "bucket_name": "qfieldcloud-local",
+                      "region_name": "",
+                      "endpoint_url": "http://172.17.0.1:8009"
+                  },
+                  "QFC_IS_LEGACY": false
+              }
+          }'
+          EOF
+
+          echo "TEST_SUITE_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env
+
       - name: Pull docker containers
         run: docker compose pull
 
@@ -97,9 +130,9 @@ jobs:
             Failed job run for branch `${{ github.head_ref || github.ref_name }}`, check ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} .
           gchat_webhook_url: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}
 
-      # - name: Setup tmate session
-      #   if: ${{ failure() }}
-      #   uses: mxschmitt/action-tmate@v3
-      #   timeout-minutes: 30
-      #   with:
-      #     limit-access-to-actor: true
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
+        with:
+          limit-access-to-actor: true

docker-app/qfieldcloud/core/cron.py

@@ -10,6 +10,8 @@
 from ..core.models import ApplyJob, ApplyJobDelta, Delta, Job, Project
 from ..core.utils2 import storage
 from .invitations_utils import send_invitation
+from qfieldcloud.filestorage.models import File
+
 
 logger = logging.getLogger(__name__)
 
@@ -115,17 +117,28 @@ def do(self):
             .values("id")
         ]
 
-        for project in projects:
-            project_id = str(project.id)
-            package_ids = storage.get_stored_package_ids(project_id)
-
-            for package_id in package_ids:
-                # keep the last package
-                if package_id == str(project.last_package_job_id):
-                    continue
-
-                # the job is still active, so it might be one of the new packages
-                if package_id in job_ids:
-                    continue
-
-                storage.delete_stored_package(project, package_id)
+        # TODO Delete with QF-4963 Drop support for legacy storage
+        if self.job.project.uses_legacy_storage:
+            for project in projects:
+                project_id = str(project.id)
+                package_ids = storage.get_stored_package_ids(project_id)
+
+                for package_id in package_ids:
+                    # keep the last package
+                    if package_id == str(project.last_package_job_id):
+                        continue
+
+                    # the job is still active, so it might be one of the new packages
+                    if package_id in job_ids:
+                        continue
+
+                    storage.delete_stored_package(project, package_id)
+        else:
+            delete_count = File.objects.filter(
+                project=self.job.project,
+                package_job_id__in=job_ids,
+            ).delete()
+
+            logger.warning(
+                f"Cron have identified and deleted {delete_count} package files from previous packages!"
+            )

docker-app/qfieldcloud/core/exceptions.py

@@ -101,6 +101,15 @@ class MultipleContentsError(QFieldCloudException):
     status_code = status.HTTP_400_BAD_REQUEST
 
 
+class ExplicitDeletionOfLastFileVersionError(QFieldCloudException):
+    """Raised when a request contains multiple files
+    (i.e. when it should contain at most one)"""
+
+    code = "explicit_deletion_of_last_version"
+    message = "Explicit deletion of last file version is not allowed!"
+    status_code = status.HTTP_400_BAD_REQUEST
+
+
 class ObjectNotFoundError(QFieldCloudException):
     """Raised when a requested object doesn't exist
     (e.g. wrong project id into the request)"""

docker-app/qfieldcloud/core/fields.py

@@ -0,0 +1,38 @@
+from django.db import models
+from django.db.models.fields.files import FieldFile
+from django.core.files.storage import storages
+
+
+class EmptyStorageNameError(Exception):
+    def __init__(self, *args: object, instance: models.Model) -> None:
+        message = f"Storage name is empty for {instance=}"
+        super().__init__(message, *args)
+
+
+class DynamicStorageFieldFile(FieldFile):
+    def __init__(self, instance, field, name):
+        super().__init__(instance, field, name)
+
+        storage_name = instance._get_file_storage_name()
+        if not storage_name:
+            raise EmptyStorageNameError(instance=instance)
+
+        self.storage = storages[storage_name]
+
+
+class DynamicStorageFileField(models.FileField):
+    attr_class = DynamicStorageFieldFile
+
+    def pre_save(self, instance, add):
+        storage_name = instance._get_file_storage_name()
+
+        if not storage_name:
+            raise EmptyStorageNameError(instance=instance)
+
+        storage = storages[storage_name]
+
+        self.storage = storage
+
+        file = super().pre_save(instance, add)
+
+        return file

docker-app/qfieldcloud/core/management/commands/dequeue.py

@@ -70,11 +70,15 @@ def handle(self, *args, **options):
                     ]
                 ).values("project_id")
 
-                # select all the pending jobs, that their project has no other active job
+                # select all the pending jobs, that their project has no other active job or `is_locked` flag is `True`
                 jobs_qs = (
                     Job.objects.select_for_update(skip_locked=True)
                     .filter(status=Job.Status.PENDING)
-                    .exclude(project_id__in=busy_projects_ids_qs)
+                    .exclude(
+                        project_id__in=busy_projects_ids_qs,
+                        # skip all projects that are currently locked, most probably because of file transfer
+                        project__is_locked=True,
+                    )
                     .order_by("created_at")
                 )