From 9c971cf734c6b0e903a03e3f6065e05d43e498d2 Mon Sep 17 00:00:00 2001 From: Mattia Date: Fri, 11 Oct 2024 12:22:33 +0200 Subject: [PATCH 1/9] [Fixes #273] connect resource with execution request --- importer/celery_tasks.py | 3 +++ importer/handlers/common/metadata.py | 2 ++ importer/handlers/common/vector.py | 6 +++--- importer/orchestrator.py | 5 +++++ importer/tests/unit/test_task.py | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/importer/celery_tasks.py b/importer/celery_tasks.py index ac79fe8..4888577 100644 --- a/importer/celery_tasks.py +++ b/importer/celery_tasks.py @@ -367,6 +367,9 @@ def create_geonode_resource( handler_module_path, resource, _exec, **kwargs ) + # assign geonode resource to ExectionRequest + orchestrator.update_execution_request_obj(_exec, {"geonode_resource": resource}) + # at the end recall the import_orchestrator for the next step import_orchestrator.apply_async( ( diff --git a/importer/handlers/common/metadata.py b/importer/handlers/common/metadata.py index 76fbbcf..b0778cd 100644 --- a/importer/handlers/common/metadata.py +++ b/importer/handlers/common/metadata.py @@ -78,6 +78,8 @@ def import_resource(self, files: dict, execution_id: str, **kwargs): self.handle_metadata_resource(_exec, dataset, original_handler) dataset.refresh_from_db() + # assign the resource to the execution_obj + orchestrator.update_execution_request_obj(_exec, {"geonode_resource": dataset}) orchestrator.evaluate_execution_progress( execution_id, handler_module_path=str(self) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 13f49cf..a18f3c0 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -607,7 +607,7 @@ def create_geonode_resource( resource_manager.set_thumbnail(None, instance=saved_dataset) ResourceBase.objects.filter(alternate=alternate).update(dirty_state=False) - + saved_dataset.refresh_from_db() return saved_dataset @@ -805,13 +805,13 @@ def _import_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs "Dynamic model does not exists, removing ogr2ogr table in progress" ) if instance_name is None: - logger.info("No table created, skipping...") + logger.warning("No table created, skipping...") return db_name = os.getenv("DEFAULT_BACKEND_DATASTORE", "datastore") with connections[db_name].cursor() as cursor: cursor.execute(f"DROP TABLE {instance_name}") except Exception as e: - logger.info(e) + logger.warning(e) pass def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs): diff --git a/importer/orchestrator.py b/importer/orchestrator.py index fb170ac..e87127e 100644 --- a/importer/orchestrator.py +++ b/importer/orchestrator.py @@ -332,6 +332,11 @@ def update_execution_request_status( task_args=celery_task_request.args ) + def update_execution_request_obj(self, _exec_obj, payload): + ExecutionRequest.objects.filter(pk=_exec_obj.pk).update(**payload) + _exec_obj.refresh_from_db() + return _exec_obj + def _last_step(self, execution_id, handler_module_path): """ Last hookable step for each handler before mark the execution as completed diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index ae1d436..3c80b85 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -245,7 +245,7 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( """ try: with self.assertRaises(Exception): - get_resource.return_falue = True + get_resource.return_value = True publish_resources.return_value = True extract_resource_to_publish.return_value = [ {"crs": 4326, "name": "dataset3"} From 55b236f8635358e7537665f0778b0c3e1ad18f5b Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:46:16 +0200 Subject: [PATCH 2/9] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b72f2d3..4f29189 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ ![PyPI - Downloads](https://img.shields.io/pypi/dm/geonode-importer) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GeoNode/geonode-importer/runtests.yml) ![GitHub top language](https://img.shields.io/github/languages/top/GeoNode/geonode-importer) - +# NOTE: +The app works with geonode <=4.4.x, in **GeoNode 5** (master branch) the importer is into the main project, so this app is no longer used https://github.com/GeoNode/geonode/issues/12368. # geonode-importer From 5401983fba06335c475a331c9ac07209cee015b3 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:55:11 +0200 Subject: [PATCH 3/9] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f29189..27f784d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ ![PyPI - Downloads](https://img.shields.io/pypi/dm/geonode-importer) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GeoNode/geonode-importer/runtests.yml) ![GitHub top language](https://img.shields.io/github/languages/top/GeoNode/geonode-importer) # NOTE: -The app works with geonode <=4.4.x, in **GeoNode 5** (master branch) the importer is into the main project, so this app is no longer used https://github.com/GeoNode/geonode/issues/12368. +GeoNode 5 (master branch) includes the importer in its core. This repository and the `geonode-importer` package will be maintained for GeoNode <= 4.4.x + +For more information https://github.com/GeoNode/geonode/issues/12368. # geonode-importer From 7cce405bd8b2479f90a476e330f8c40b473edfb2 Mon Sep 17 00:00:00 2001 From: Mattia Date: Tue, 22 Oct 2024 16:10:40 +0200 Subject: [PATCH 4/9] store spacial file always true for cloning --- importer/handlers/common/raster.py | 2 +- importer/handlers/common/remote.py | 2 +- importer/handlers/common/vector.py | 2 +- importer/handlers/shapefile/handler.py | 2 +- importer/handlers/tiles3d/handler.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index 350d877..1a02b3d 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -104,7 +104,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/common/remote.py b/importer/handlers/common/remote.py index ad9f997..59a1cfa 100755 --- a/importer/handlers/common/remote.py +++ b/importer/handlers/common/remote.py @@ -84,7 +84,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "source": _data.pop("source", "upload"), diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index a18f3c0..2b68b53 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -114,7 +114,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/shapefile/handler.py b/importer/handlers/shapefile/handler.py index 2ace0bf..7f4acfc 100644 --- a/importer/handlers/shapefile/handler.py +++ b/importer/handlers/shapefile/handler.py @@ -85,7 +85,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data additional_params = { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index 5468941..f7e58fb 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -138,7 +138,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), From 7af997ab154e70cd9036c20db23a2f6555371d7e Mon Sep 17 00:00:00 2001 From: Mattia Date: Tue, 22 Oct 2024 16:16:13 +0200 Subject: [PATCH 5/9] store spacial file always true for cloning --- importer/handlers/common/vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 2b68b53..85998fa 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -220,7 +220,7 @@ def perform_last_step(execution_id): that the execution is completed """ _exec = BaseHandler.perform_last_step(execution_id=execution_id) - if _exec and not _exec.input_params.get("store_spatial_file", False): + if _exec and not _exec.input_params.get("store_spatial_file", True): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) # getting all assets list assets = filter(None, [get_default_asset(x.resource) for x in resources]) From b273faa311732696ac9e6b419117dfa311355b40 Mon Sep 17 00:00:00 2001 From: Mattia Date: Tue, 22 Oct 2024 17:22:54 +0200 Subject: [PATCH 6/9] fix build --- Dockerfile | 2 +- importer/handlers/tiles3d/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b40b28c..86e2e59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM geonode/geonode-base:latest-ubuntu-22.04 RUN rm -rf /usr/src/geonode RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode -RUN cd /usr/src/geonode && git fetch --all && git checkout master && cd - +RUN cd /usr/src/geonode && git fetch --all && git checkout 4.4.x && cd - RUN mkdir -p /usr/src/importer RUN cd .. diff --git a/importer/handlers/tiles3d/tests.py b/importer/handlers/tiles3d/tests.py index 8817b86..aee5595 100755 --- a/importer/handlers/tiles3d/tests.py +++ b/importer/handlers/tiles3d/tests.py @@ -91,7 +91,7 @@ def test_extract_params_from_data(self): action="copy", ) - self.assertEqual(actual, {"title": "title_of_the_cloned_resource"}) + self.assertEqual(actual, {'store_spatial_file': True, 'title': 'title_of_the_cloned_resource'}) def test_is_valid_should_raise_exception_if_the_3dtiles_is_invalid(self): data = { From fe4ccb5534e7dce8d61499efba3cdd5c50cee4a5 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:19:10 +0100 Subject: [PATCH 7/9] =?UTF-8?q?[Fixes=20#12763]=203D=20tiles=20geometricEr?= =?UTF-8?q?ror=20mandatory=20field=20should=20be=20on=20t=E2=80=A6=20(#279?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Fixes #12763] 3D tiles geometricError mandatory field should be on tileset level ref to https://github.com/GeoNode/geonode/issues/12763 * fix tests * Update tests.py * Update test_end2end.py * fix test --- importer/handlers/tiles3d/handler.py | 2 +- importer/handlers/tiles3d/tests.py | 5 ++--- importer/tests/end2end/test_end2end.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index f7e58fb..b727363 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -124,7 +124,7 @@ def validate_3dtile_payload(payload): "The mandatory 'boundingVolume' for the key 'root' is missing" ) - error = payload.get("root", {}).get("geometricError", None) + error = payload.get("geometricError", None) or payload.get("root", {}).get("geometricError", None) if error is None: raise Invalid3DTilesException( "The mandatory 'geometricError' for the key 'root' is missing" diff --git a/importer/handlers/tiles3d/tests.py b/importer/handlers/tiles3d/tests.py index aee5595..52ae9b6 100755 --- a/importer/handlers/tiles3d/tests.py +++ b/importer/handlers/tiles3d/tests.py @@ -140,7 +140,7 @@ def test_validate_should_raise_exception_for_invalid_asset_key(self): def test_validate_should_raise_exception_for_invalid_root_boundingVolume(self): _json = { "asset": {"version": "1.1"}, - "geometricError": 1.0, + "geometricError": 1.0, "root": {"foo": {"box": []}, "geometricError": 0.0}, } _path = "/tmp/tileset.json" @@ -159,7 +159,6 @@ def test_validate_should_raise_exception_for_invalid_root_boundingVolume(self): def test_validate_should_raise_exception_for_invalid_root_geometricError(self): _json = { "asset": {"version": "1.1"}, - "geometricError": 1.0, "root": {"boundingVolume": {"box": []}, "foo": 0.0}, } _path = "/tmp/tileset.json" @@ -170,7 +169,7 @@ def test_validate_should_raise_exception_for_invalid_root_geometricError(self): self.assertIsNotNone(_exc) self.assertTrue( - "The mandatory 'geometricError' for the key 'root' is missing" + "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'" in str(_exc.exception.detail) ) os.remove(_path) diff --git a/importer/tests/end2end/test_end2end.py b/importer/tests/end2end/test_end2end.py index db8225b..85c6baf 100644 --- a/importer/tests/end2end/test_end2end.py +++ b/importer/tests/end2end/test_end2end.py @@ -543,7 +543,7 @@ def test_import_wms(self): "lookup": resource_to_take, "parse_remote_metadata": True, } - initial_name = res.title + initial_name = res.title.lower().replace(" ", "_") assert_payload = { "subtype": "remote", "title": res.title, From f9e7a74819956d8e1f4d8a3bb385a2a02d2e3fff Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Wed, 15 Jan 2025 18:25:37 +0100 Subject: [PATCH 8/9] [Fixes #12789] Improve 3dtiles filename handling (#281) --- importer/handlers/tiles3d/handler.py | 33 +++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index b727363..b0ff203 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -60,8 +60,7 @@ def can_handle(_data) -> bool: if not base: return False ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1] - input_filename = os.path.basename(base if isinstance(base, str) else base.name) - if ext in ["json"] and "tileset.json" in input_filename: + if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base): return True return False @@ -90,18 +89,7 @@ def is_valid(files, user): ) try: - with open(_file, "r") as _readed_file: - _file = json.loads(_readed_file.read()) - # required key described in the specification of 3dtiles - # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92 - is_valid = all( - key in _file.keys() for key in ("asset", "geometricError", "root") - ) - - if not is_valid: - raise Invalid3DTilesException( - "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'" - ) + _file = Tiles3DFileHandler.is_3dtiles_json(_file) Tiles3DFileHandler.validate_3dtile_payload(payload=_file) @@ -109,6 +97,21 @@ def is_valid(files, user): raise Invalid3DTilesException(e) return True + + @staticmethod + def is_3dtiles_json(_file): + with open(_file, "r") as _readed_file: + _file = json.loads(_readed_file.read()) + # required key described in the specification of 3dtiles + # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92 + is_valid = all(key in _file.keys() for key in ("asset", "geometricError", "root")) + + if not is_valid: + raise Invalid3DTilesException( + "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'" + ) + + return _file @staticmethod def validate_3dtile_payload(payload): @@ -212,7 +215,7 @@ def create_geonode_resource( asset=None, ): # we want just the tileset.json as location of the asset - asset.location = [path for path in asset.location if "tileset.json" in path] + asset.location = [path for path in asset.location if path.endswith(".json")] asset.save() resource = super().create_geonode_resource( From 3231f58b9cdc5a0d840b497c45288c56668bb991 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:00:19 +0100 Subject: [PATCH 9/9] Fix migrations for asset (#283) * Fix migrations for create handlerinfo via asset * Fix migrations for create handlerinfo via asset --- importer/handlers/tiles3d/handler.py | 13 +++-- importer/migrations/0006_dataset_migration.py | 2 +- .../0007_align_resourcehandler_with_asset.py | 54 +++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 importer/migrations/0007_align_resourcehandler_with_asset.py diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index b0ff203..7e5d525 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -56,12 +56,15 @@ def can_handle(_data) -> bool: This endpoint will return True or False if with the info provided the handler is able to handle the file or not """ - base = _data.get("base_file") - if not base: + try: + base = _data.get("base_file") + if not base: + return False + ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1] + if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base): + return True + except Exception: return False - ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1] - if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base): - return True return False @staticmethod diff --git a/importer/migrations/0006_dataset_migration.py b/importer/migrations/0006_dataset_migration.py index 768d6a4..840b95c 100644 --- a/importer/migrations/0006_dataset_migration.py +++ b/importer/migrations/0006_dataset_migration.py @@ -11,7 +11,7 @@ def dataset_migration(apps, _): pk__in=NewResources.objects.values_list("resource_id", flat=True) ).exclude(subtype__in=["remote", None]): # generating orchestrator expected data file - if not old_resource.files: + if not hasattr(old_resource, "files"): if old_resource.is_vector(): converted_files = [{"base_file": "placeholder.shp"}] else: diff --git a/importer/migrations/0007_align_resourcehandler_with_asset.py b/importer/migrations/0007_align_resourcehandler_with_asset.py new file mode 100644 index 0000000..7ca9b9c --- /dev/null +++ b/importer/migrations/0007_align_resourcehandler_with_asset.py @@ -0,0 +1,54 @@ +# Generated by Django 3.2.15 on 2022-10-04 13:03 + +import logging +from django.db import migrations +from importer.orchestrator import orchestrator +from geonode.layers.models import Dataset +from geonode.assets.utils import get_default_asset +from geonode.utils import get_allowed_extensions + +logger = logging.getLogger("django") + +def dataset_migration(apps, _): + NewResources = apps.get_model("importer", "ResourceHandlerInfo") + for old_resource in Dataset.objects.exclude( + pk__in=NewResources.objects.values_list("resource_id", flat=True) + ).exclude(subtype__in=["remote", None]): + # generating orchestrator expected data file + if old_resource.resourcehandlerinfo_set.first() is None: + if get_default_asset(old_resource): + available_choices = get_allowed_extensions() + not_main_files = ["xml", "sld", "zip", "kmz"] + base_file_choices = set(x for x in available_choices if x not in not_main_files) + output_files = dict() + for _file in get_default_asset(old_resource).location: + if _file.split(".")[-1] in base_file_choices: + output_files.update({"base_file": _file}) + break + else: + if old_resource.is_vector(): + output_files = {"base_file": "placeholder.shp"} + else: + output_files = {"base_file": "placeholder.tiff"} + + handler = orchestrator.get_handler(output_files) + if handler is None: + logger.error(f"Handler not found for resource: {old_resource}") + continue + handler.create_resourcehandlerinfo( + handler_module_path=str(handler), + resource=old_resource, + execution_id=None + ) + else: + logger.debug(f"resourcehandler info already exists for the resource") + + +class Migration(migrations.Migration): + dependencies = [ + ("importer", "0006_dataset_migration"), + ] + + operations = [ + migrations.RunPython(dataset_migration), + ]