Merge pull request #1078 from opengisch/pass_project_not_only_id

[refactor] Make functions in `utils2.storages` us the `Project` instance, not only the `project_id`

docker-app/qfieldcloud/core/cron.py

@@ -128,4 +128,4 @@ def do(self):
                 if package_id in job_ids:
                     continue
 
-                storage.delete_stored_package(project_id, package_id)
+                storage.delete_stored_package(project, package_id)

docker-app/qfieldcloud/core/models.py

@@ -1417,7 +1417,7 @@ def save(self, recompute_storage=False, *args, **kwargs):
         logger.debug(f"Saving project {self}...")
 
         if recompute_storage:
-            self.file_storage_bytes = storage.get_project_file_storage_in_bytes(self.id)
+            self.file_storage_bytes = storage.get_project_file_storage_in_bytes(self)
 
         # Ensure that the Project's storage_keep_versions is at least 1, and reflects the plan's default storage_keep_versions value.
         if not self.storage_keep_versions:

docker-app/qfieldcloud/core/tests/test_packages.py

@@ -665,7 +665,7 @@ def test_outdated_packaged_files_are_deleted(self):
         old_package = PackageJob.objects.filter(project=self.project1).latest(
             "created_at"
         )
-        stored_package_ids = get_stored_package_ids(self.project1.id)
+        stored_package_ids = get_stored_package_ids(self.project1)
         self.assertIn(str(old_package.id), stored_package_ids)
         self.assertEqual(len(stored_package_ids), 1)
 
@@ -683,7 +683,7 @@ def test_outdated_packaged_files_are_deleted(self):
             "created_at"
         )
 
-        stored_package_ids = get_stored_package_ids(self.project1.id)
+        stored_package_ids = get_stored_package_ids(self.project1)
 
         self.assertNotEqual(old_package.id, new_package.id)
         self.assertNotIn(str(old_package.id), stored_package_ids)

docker-app/qfieldcloud/core/utils2/storage.py

@@ -451,6 +451,14 @@ def upload_project_file(
 
 
 def delete_all_project_files_permanently(project_id: str) -> None:
+    """Deletes all project files permanently.
+
+    Args:
+        project_id (str): the project which files shall be deleted. Note that the `project_id` might be a of a already deleted project which files are still dangling around.
+
+    Raises:
+        RuntimeError: if the produced Object Storage key to delete is not in the right format
+    """
     prefix = f"projects/{project_id}/"
 
     if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", prefix):
@@ -466,7 +474,9 @@ def delete_project_file_permanently(
 ):  # noqa: F821
     logger.info(f"Requested delete (permanent) of project file {filename=}")
 
-    file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename)
+    file = qfieldcloud.core.utils.get_project_file_with_versions(
+        str(project.id), filename
+    )
 
     if not file:
         raise Exception(
@@ -510,7 +520,7 @@ def delete_project_file_permanently(
 
 
 def delete_project_file_version_permanently(
-    project: qfieldcloud.core.models.Project,  # noqa: F821
+    project: qfieldcloud.core.models.Project,
     filename: str,
     version_id: str,
     include_older: bool = False,
@@ -526,7 +536,8 @@ def delete_project_file_version_permanently(
     Returns:
         int: the number of versions deleted
     """
-    file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename)
+    project_id = str(project.id)
+    file = qfieldcloud.core.utils.get_project_file_with_versions(project_id, filename)
 
     if not file:
         raise Exception(
@@ -591,7 +602,8 @@ def delete_project_file_version_permanently(
     return versions_to_delete
 
 
-def get_stored_package_ids(project_id: str) -> set[str]:
+def get_stored_package_ids(project: qfieldcloud.core.models.Project) -> set[str]:
+    project_id = project.id
     bucket = qfieldcloud.core.utils.get_s3_bucket()
     prefix = f"projects/{project_id}/packages/"
     root_path = PurePath(prefix)
@@ -605,7 +617,10 @@ def get_stored_package_ids(project_id: str) -> set[str]:
     return package_ids
 
 
-def delete_stored_package(project_id: str, package_id: str) -> None:
+def delete_stored_package(
+    project: qfieldcloud.core.models.Project, package_id: str
+) -> None:
+    project_id = str(project.id)
     prefix = f"projects/{project_id}/packages/{package_id}/"
 
     if not re.match(
@@ -620,11 +635,12 @@ def delete_stored_package(project_id: str, package_id: str) -> None:
     _delete_by_prefix_permanently(prefix)
 
 
-def get_project_file_storage_in_bytes(project_id: str) -> int:
+def get_project_file_storage_in_bytes(project: qfieldcloud.core.models.Project) -> int:
     """Calculates the project files storage in bytes, including their versions.
 
     WARNING This function can be quite slow on projects with thousands of files.
     """
+    project_id = str(project.id)
     bucket = qfieldcloud.core.utils.get_s3_bucket()
     total_bytes = 0
     prefix = f"projects/{project_id}/files/"

docker-app/worker_wrapper/wrapper.py

@@ -408,8 +408,7 @@ def after_docker_run(self) -> None:
         )
 
         try:
-            project_id = str(self.job.project.id)
-            package_ids = storage.get_stored_package_ids(project_id)
+            package_ids = storage.get_stored_package_ids(self.job.project)
             job_ids = [
                 str(job["id"])
                 for job in Job.objects.filter(
@@ -431,7 +430,7 @@ def after_docker_run(self) -> None:
                 if package_id in job_ids:
                     continue
 
-                storage.delete_stored_package(project_id, package_id)
+                storage.delete_stored_package(self.job.project, package_id)
         except Exception as err:
             logger.error(
                 "Failed to delete dangling packages, will be deleted via CRON later.",