diff --git a/docker-app/qfieldcloud/core/cron.py b/docker-app/qfieldcloud/core/cron.py index 9b8653763..500903009 100644 --- a/docker-app/qfieldcloud/core/cron.py +++ b/docker-app/qfieldcloud/core/cron.py @@ -128,4 +128,4 @@ def do(self): if package_id in job_ids: continue - storage.delete_stored_package(project_id, package_id) + storage.delete_stored_package(project, package_id) diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index 6957c4288..2feb96dba 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -1417,7 +1417,7 @@ def save(self, recompute_storage=False, *args, **kwargs): logger.debug(f"Saving project {self}...") if recompute_storage: - self.file_storage_bytes = storage.get_project_file_storage_in_bytes(self.id) + self.file_storage_bytes = storage.get_project_file_storage_in_bytes(self) # Ensure that the Project's storage_keep_versions is at least 1, and reflects the plan's default storage_keep_versions value. if not self.storage_keep_versions: diff --git a/docker-app/qfieldcloud/core/tests/test_packages.py b/docker-app/qfieldcloud/core/tests/test_packages.py index 7906d82c8..3e07c1a77 100644 --- a/docker-app/qfieldcloud/core/tests/test_packages.py +++ b/docker-app/qfieldcloud/core/tests/test_packages.py @@ -665,7 +665,7 @@ def test_outdated_packaged_files_are_deleted(self): old_package = PackageJob.objects.filter(project=self.project1).latest( "created_at" ) - stored_package_ids = get_stored_package_ids(self.project1.id) + stored_package_ids = get_stored_package_ids(self.project1) self.assertIn(str(old_package.id), stored_package_ids) self.assertEqual(len(stored_package_ids), 1) @@ -683,7 +683,7 @@ def test_outdated_packaged_files_are_deleted(self): "created_at" ) - stored_package_ids = get_stored_package_ids(self.project1.id) + stored_package_ids = get_stored_package_ids(self.project1) self.assertNotEqual(old_package.id, new_package.id) self.assertNotIn(str(old_package.id), stored_package_ids) diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index d161c6e8b..e4d53356b 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -451,6 +451,14 @@ def upload_project_file( def delete_all_project_files_permanently(project_id: str) -> None: + """Deletes all project files permanently. + + Args: + project_id (str): the project which files shall be deleted. Note that the `project_id` might be a of a already deleted project which files are still dangling around. + + Raises: + RuntimeError: if the produced Object Storage key to delete is not in the right format + """ prefix = f"projects/{project_id}/" if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", prefix): @@ -466,7 +474,9 @@ def delete_project_file_permanently( ): # noqa: F821 logger.info(f"Requested delete (permanent) of project file {filename=}") - file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename) + file = qfieldcloud.core.utils.get_project_file_with_versions( + str(project.id), filename + ) if not file: raise Exception( @@ -510,7 +520,7 @@ def delete_project_file_permanently( def delete_project_file_version_permanently( - project: qfieldcloud.core.models.Project, # noqa: F821 + project: qfieldcloud.core.models.Project, filename: str, version_id: str, include_older: bool = False, @@ -526,7 +536,8 @@ def delete_project_file_version_permanently( Returns: int: the number of versions deleted """ - file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename) + project_id = str(project.id) + file = qfieldcloud.core.utils.get_project_file_with_versions(project_id, filename) if not file: raise Exception( @@ -591,7 +602,8 @@ def delete_project_file_version_permanently( return versions_to_delete -def get_stored_package_ids(project_id: str) -> set[str]: +def get_stored_package_ids(project: qfieldcloud.core.models.Project) -> set[str]: + project_id = project.id bucket = qfieldcloud.core.utils.get_s3_bucket() prefix = f"projects/{project_id}/packages/" root_path = PurePath(prefix) @@ -605,7 +617,10 @@ def get_stored_package_ids(project_id: str) -> set[str]: return package_ids -def delete_stored_package(project_id: str, package_id: str) -> None: +def delete_stored_package( + project: qfieldcloud.core.models.Project, package_id: str +) -> None: + project_id = str(project.id) prefix = f"projects/{project_id}/packages/{package_id}/" if not re.match( @@ -620,11 +635,12 @@ def delete_stored_package(project_id: str, package_id: str) -> None: _delete_by_prefix_permanently(prefix) -def get_project_file_storage_in_bytes(project_id: str) -> int: +def get_project_file_storage_in_bytes(project: qfieldcloud.core.models.Project) -> int: """Calculates the project files storage in bytes, including their versions. WARNING This function can be quite slow on projects with thousands of files. """ + project_id = str(project.id) bucket = qfieldcloud.core.utils.get_s3_bucket() total_bytes = 0 prefix = f"projects/{project_id}/files/" diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 4f0108226..b76862ad0 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -408,8 +408,7 @@ def after_docker_run(self) -> None: ) try: - project_id = str(self.job.project.id) - package_ids = storage.get_stored_package_ids(project_id) + package_ids = storage.get_stored_package_ids(self.job.project) job_ids = [ str(job["id"]) for job in Job.objects.filter( @@ -431,7 +430,7 @@ def after_docker_run(self) -> None: if package_id in job_ids: continue - storage.delete_stored_package(project_id, package_id) + storage.delete_stored_package(self.job.project, package_id) except Exception as err: logger.error( "Failed to delete dangling packages, will be deleted via CRON later.",