align 11x with master (#284)

* [Fixes #273] connect resource with execution request

* Update README.md

* store spacial file always true for cloning
* fix build

* [Fixes #12763] 3D tiles geometricError mandatory field should be on t… (#279)

* [Fixes #12763] 3D tiles geometricError mandatory field should be on tileset level

ref to GeoNode/geonode#12763

* fix tests

* Update tests.py

* Update test_end2end.py

* fix test

* [Fixes #12789] Improve 3dtiles filename handling (#281)

* Fix migrations for asset (#283)

* Fix migrations for create handlerinfo via asset

* Fix migrations for create handlerinfo via asset

---------

Co-authored-by: Giovanni Allegri <giovanni.allegri@gmail.com>

Author: mattiagiupponi

importer/handlers/tiles3d/handler.py

@@ -56,13 +56,15 @@ def can_handle(_data) -> bool:
         This endpoint will return True or False if with the info provided
         the handler is able to handle the file or not
         """
-        base = _data.get("base_file")
-        if not base:
+        try:
+            base = _data.get("base_file")
+            if not base:
+                return False
+            ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1]
+            if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base):
+                return True
+        except Exception:
             return False
-        ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1]
-        input_filename = os.path.basename(base if isinstance(base, str) else base.name)
-        if ext in ["json"] and "tileset.json" in input_filename:
-            return True
         return False
 
     @staticmethod
@@ -90,25 +92,29 @@ def is_valid(files, user):
             )
 
         try:
-            with open(_file, "r") as _readed_file:
-                _file = json.loads(_readed_file.read())
-            # required key described in the specification of 3dtiles
-            # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92
-            is_valid = all(
-                key in _file.keys() for key in ("asset", "geometricError", "root")
-            )
-
-            if not is_valid:
-                raise Invalid3DTilesException(
-                    "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'"
-                )
+            _file = Tiles3DFileHandler.is_3dtiles_json(_file)
 
             Tiles3DFileHandler.validate_3dtile_payload(payload=_file)
 
         except Exception as e:
             raise Invalid3DTilesException(e)
 
         return True
+        
+    @staticmethod
+    def is_3dtiles_json(_file):
+        with open(_file, "r") as _readed_file:
+            _file = json.loads(_readed_file.read())
+            # required key described in the specification of 3dtiles
+            # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92
+        is_valid = all(key in _file.keys() for key in ("asset", "geometricError", "root"))
+
+        if not is_valid:
+            raise Invalid3DTilesException(
+                "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'"
+            )
+
+        return _file
 
     @staticmethod
     def validate_3dtile_payload(payload):
@@ -212,7 +218,7 @@ def create_geonode_resource(
         asset=None,
     ):
         # we want just the tileset.json as location of the asset
-        asset.location = [path for path in asset.location if "tileset.json" in path]
+        asset.location = [path for path in asset.location if path.endswith(".json")]
         asset.save()
 
         resource = super().create_geonode_resource(

importer/migrations/0006_dataset_migration.py

@@ -11,7 +11,7 @@ def dataset_migration(apps, _):
         pk__in=NewResources.objects.values_list("resource_id", flat=True)
     ).exclude(subtype__in=["remote", None]):
         # generating orchestrator expected data file
-        if not old_resource.files:
+        if not hasattr(old_resource, "files"):
             if old_resource.is_vector():
                 converted_files = [{"base_file": "placeholder.shp"}]
             else:

importer/migrations/0007_align_resourcehandler_with_asset.py

@@ -0,0 +1,54 @@
+# Generated by Django 3.2.15 on 2022-10-04 13:03
+
+import logging
+from django.db import migrations
+from importer.orchestrator import orchestrator
+from geonode.layers.models import Dataset
+from geonode.assets.utils import get_default_asset
+from geonode.utils import get_allowed_extensions
+
+logger = logging.getLogger("django")
+
+def dataset_migration(apps, _):
+    NewResources = apps.get_model("importer", "ResourceHandlerInfo")
+    for old_resource in Dataset.objects.exclude(
+        pk__in=NewResources.objects.values_list("resource_id", flat=True)
+    ).exclude(subtype__in=["remote", None]):
+        # generating orchestrator expected data file
+        if old_resource.resourcehandlerinfo_set.first() is None:
+            if get_default_asset(old_resource):
+                available_choices = get_allowed_extensions()
+                not_main_files = ["xml", "sld", "zip", "kmz"]
+                base_file_choices = set(x for x in available_choices if x not in not_main_files)
+                output_files = dict()
+                for _file in get_default_asset(old_resource).location:
+                    if _file.split(".")[-1] in base_file_choices:
+                        output_files.update({"base_file": _file})
+                        break
+            else:
+                if old_resource.is_vector():
+                    output_files = {"base_file": "placeholder.shp"}
+                else:
+                    output_files = {"base_file": "placeholder.tiff"}
+
+            handler = orchestrator.get_handler(output_files)
+            if handler is None:
+                logger.error(f"Handler not found for resource: {old_resource}")
+                continue
+            handler.create_resourcehandlerinfo(
+                handler_module_path=str(handler),
+                resource=old_resource,
+                execution_id=None
+            )   
+        else:
+            logger.debug(f"resourcehandler info already exists for the resource")  
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("importer", "0006_dataset_migration"),
+    ]
+
+    operations = [
+        migrations.RunPython(dataset_migration),
+    ]