[Fixes #261] Add remote WMS importer handler (#263)

* [Fixes #261] Add remote WMS importer handler
* Add handler in settings

Author: mattiagiupponi

importer/handlers/base.py

@@ -255,11 +255,12 @@ def get_ogr2ogr_task_group(
         """
         return NotImplementedError
 
-    def delete_resource(self, instance):
+    @staticmethod
+    def delete_resource(instance):
         """
         Base function to delete the resource with all the dependencies (example: dynamic model)
         """
-        return NotImplementedError
+        return
 
     def _get_execution_request_object(self, execution_id: str):
         return ExecutionRequest.objects.filter(exec_id=execution_id).first()
@@ -326,13 +327,15 @@ def rollback(
     def _create_geonode_resource_rollback(
         self, exec_id, istance_name=None, *args, **kwargs
     ):
+        from importer.orchestrator import orchestrator
         """
         The handler will remove the resource from geonode
         """
         logger.info(
             f"Rollback geonode step in progress for execid: {exec_id} resource created was: {istance_name}"
         )
-        resource = ResourceBase.objects.filter(alternate__icontains=istance_name)
+        _exec_obj = orchestrator.get_execution_object(exec_id)
+        resource = ResourceBase.objects.filter(alternate__icontains=istance_name, owner=_exec_obj.user)
         if resource.exists():
             resource.delete()
 

importer/handlers/common/raster.py

@@ -370,10 +370,11 @@ def overwrite_geonode_resource(
         resource_type: Dataset = Dataset,
         asset=None,
     ):
-        dataset = resource_type.objects.filter(alternate__icontains=alternate)
-
+        
         _exec = self._get_execution_request_object(execution_id)
 
+        dataset = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
+
         _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue

importer/handlers/common/remote.py

@@ -91,6 +91,7 @@ def extract_params_from_data(_data, action=None):
             "title": _data.pop("title", None),
             "url": _data.pop("url", None),
             "type": _data.pop("type", None),
+            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
         }, _data
 
     def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
@@ -110,28 +111,29 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
         try:
             params = _exec.input_params.copy()
             url = params.get("url")
-            title = params.get("title", os.path.basename(urlparse(url).path))
+            title = params.get("title", None) or os.path.basename(urlparse(url).path)
 
             # start looping on the layers available
             layer_name = self.fixup_name(title)
 
             should_be_overwritten = _exec.input_params.get("overwrite_existing_layer")
 
+            payload_alternate = params.get("remote_resource_id", None)
+
             user_datasets = ResourceBase.objects.filter(
-                owner=_exec.user, alternate=layer_name
+                owner=_exec.user, alternate=payload_alternate or layer_name
             )
 
             dataset_exists = user_datasets.exists()
 
-            if dataset_exists and should_be_overwritten:
-                layer_name, alternate = (
-                    layer_name,
-                    user_datasets.first().alternate.split(":")[-1],
-                )
-            elif not dataset_exists:
-                alternate = layer_name
-            else:
-                alternate = create_alternate(layer_name, execution_id)
+            layer_name, alternate = self.generate_alternate(
+                layer_name,
+                execution_id,
+                should_be_overwritten,
+                payload_alternate,
+                user_datasets,
+                dataset_exists,
+            )
 
             import_orchestrator.apply_async(
                 (
@@ -150,12 +152,32 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
             logger.error(e)
             raise e
 
+    def generate_alternate(
+        self,
+        layer_name,
+        execution_id,
+        should_be_overwritten,
+        payload_alternate,
+        user_datasets,
+        dataset_exists,
+    ):
+        if dataset_exists and should_be_overwritten:
+            layer_name, alternate = (
+                payload_alternate or layer_name,
+                user_datasets.first().alternate.split(":")[-1],
+            )
+        elif not dataset_exists:
+            alternate = payload_alternate or layer_name
+        else:
+            alternate = create_alternate(payload_alternate or layer_name, execution_id)
+        return layer_name, alternate
+
     def create_geonode_resource(
         self,
         layer_name: str,
         alternate: str,
         execution_id: str,
-        resource_type: Dataset = ...,
+        resource_type: ResourceBase = ResourceBase,
         asset=None,
     ):
         """
@@ -166,19 +188,12 @@ def create_geonode_resource(
         """
         _exec = orchestrator.get_execution_object(execution_id)
         params = _exec.input_params.copy()
-        subtype = params.get("type")
 
         resource = resource_manager.create(
             None,
-            resource_type=ResourceBase,
-            defaults=dict(
-                resource_type="dataset",
-                subtype=subtype,
-                sourcetype=SOURCE_TYPE_REMOTE,
-                alternate=alternate,
-                dirty_state=True,
-                title=params.get("title", layer_name),
-                owner=_exec.user,
+            resource_type=resource_type,
+            defaults=self.generate_resource_payload(
+                layer_name, alternate, asset, _exec, None, **params
             ),
         )
         resource_manager.set_thumbnail(None, instance=resource)
@@ -217,3 +232,53 @@ def create_resourcehandlerinfo(
             execution_request=execution_id,
             kwargs=kwargs.get("kwargs", {}) or kwargs,
         )
+
+    def generate_resource_payload(
+        self, layer_name, alternate, asset, _exec, workspace, **kwargs
+    ):
+        return dict(
+            subtype=kwargs.get("type"),
+            sourcetype=SOURCE_TYPE_REMOTE,
+            alternate=alternate,
+            dirty_state=True,
+            title=kwargs.get("title", layer_name),
+            owner=_exec.user,
+        )
+
+    def overwrite_geonode_resource(
+        self,
+        layer_name: str,
+        alternate: str,
+        execution_id: str,
+        resource_type: Dataset = ResourceBase,
+        asset=None,
+    ):
+        _exec = self._get_execution_request_object(execution_id)
+        resource = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
+
+        _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
+        # if the layer exists, we just update the information of the dataset by
+        # let it recreate the catalogue
+        if resource.exists() and _overwrite:
+            resource = resource.first()
+
+            resource = resource_manager.update(
+                resource.uuid, instance=resource
+            )
+            resource_manager.set_thumbnail(
+                resource.uuid, instance=resource, overwrite=True
+            )
+            resource.refresh_from_db()
+            return resource
+        elif not resource.exists() and _overwrite:
+            logger.warning(
+                f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
+            )
+            return self.create_geonode_resource(
+                layer_name, alternate, execution_id, resource_type, asset
+            )
+        elif not resource.exists() and not _overwrite:
+            logger.warning(
+                "The resource does not exists, please use 'create_geonode_resource' to create one"
+            )
+        return

importer/handlers/common/serializer.py

@@ -13,9 +13,19 @@ class Meta:
             "title",
             "type",
             "source",
+            "overwrite_existing_layer"
         )
 
-    url = serializers.URLField(required=True)
-    title = serializers.CharField(required=False)
-    type = serializers.CharField(required=True)
+    url = serializers.URLField(
+        required=True, help_text="URL of the remote service / resource"
+    )
+    title = serializers.CharField(
+        required=True, help_text="Title of the resource. Can be None or Empty"
+    )
+    type = serializers.CharField(
+        required=True,
+        help_text="Remote resource type, for example wms or 3dtiles. Is used by the handler to understand if can handle the resource",
+    )
     source = serializers.CharField(required=False, default="upload")
+
+    overwrite_existing_layer = serializers.BooleanField(required=False, default=False)

importer/handlers/common/test_remote.py

@@ -7,6 +7,7 @@
 from importer.orchestrator import orchestrator
 from geonode.base.populate_test_data import create_single_dataset
 from geonode.resource.models import ExecutionRequest
+from geonode.base.models import ResourceBase
 
 
 class TestBaseRemoteResourceHandler(TestCase):
@@ -131,7 +132,7 @@ def test_create_geonode_resource(self):
             "layername",
             "layeralternate",
             execution_id=exec_id,
-            resource_type="ResourceBase",
+            resource_type=ResourceBase,
             asset=None,
         )
         self.assertIsNotNone(resource)

importer/handlers/common/vector.py

@@ -35,6 +35,7 @@
 from importer.orchestrator import orchestrator
 from django.db.models import Q
 import pyproj
+from geonode.geoserver.security import delete_dataset_cache, set_geowebcache_invalidate_cache
 
 logger = logging.getLogger(__name__)
 
@@ -631,16 +632,19 @@ def overwrite_geonode_resource(
         resource_type: Dataset = Dataset,
         asset=None,
     ):
-        dataset = resource_type.objects.filter(alternate__icontains=alternate)
-
         _exec = self._get_execution_request_object(execution_id)
 
+        dataset = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
+
         _overwrite = _exec.input_params.get("overwrite_existing_layer", False)
         # if the layer exists, we just update the information of the dataset by
         # let it recreate the catalogue
         if dataset.exists() and _overwrite:
             dataset = dataset.first()
 
+            delete_dataset_cache(dataset.alternate)
+            set_geowebcache_invalidate_cache(dataset.typename)
+
             dataset = resource_manager.update(
                 dataset.uuid, instance=dataset, files=asset.location
             )

importer/handlers/remote/serializers/__init__.py

@@ -0,0 +1,17 @@
+from rest_framework import serializers
+from importer.handlers.common.serializer import RemoteResourceSerializer
+
+
+class RemoteWMSSerializer(RemoteResourceSerializer):
+    class Meta:
+        model = RemoteResourceSerializer.Meta.model
+        ref_name = "RemoteWMSSerializer"
+        fields = RemoteResourceSerializer.Meta.fields + (
+            "lookup",
+            "bbox",
+            "parse_remote_metadata",
+        )
+
+    lookup = serializers.CharField(required=True)
+    bbox = serializers.ListField(required=False)
+    parse_remote_metadata = serializers.BooleanField(required=False, default=False)

importer/handlers/remote/serializers/wms.py

@@ -8,6 +8,7 @@
 from importer.orchestrator import orchestrator
 from geonode.base.populate_test_data import create_single_dataset
 from geonode.resource.models import ExecutionRequest
+from geonode.base.models import ResourceBase
 
 
 class TestRemoteTiles3DFileHandler(TestCase):
@@ -133,7 +134,7 @@ def test_create_geonode_resource_raise_error_if_url_is_not_reachabel(self):
                 "layername",
                 "layeralternate",
                 execution_id=exec_id,
-                resource_type="ResourceBase",
+                resource_type=ResourceBase,
                 asset=None,
             )
 
@@ -153,7 +154,7 @@ def test_create_geonode_resource(self):
             "layername",
             "layeralternate",
             execution_id=exec_id,
-            resource_type="ResourceBase",
+            resource_type=ResourceBase,
             asset=None,
         )
         self.assertIsNotNone(resource)