[Core] Add support for choosing default resources integration will create dynamically (#1129)

# Description

What - Add the option to create custom type of resources for an
integration

Why - Ocean Saas is the only type of Ocean integrations that can use
secrets

How - added a new config parameter to the integration settings

## Type of change

Please leave one option from the following and delete the rest:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

<h4> All tests should be run against the port production
environment(using a testing org). </h4>

### Core testing checklist

- [x] Integration able to create all default resources from scratch
- [x] Resync finishes successfully
- [x] Resync able to create entities
- [x] Resync able to update entities
- [x] Resync able to detect and delete entities
- [ ] Scheduled resync able to abort existing resync and start a new one
- [ ] Tested with at least 2 integrations from scratch
- [ ] Tested with Kafka and Polling event listeners
- [ ] Tested deletion of entities that don't pass the selector


### Integration testing checklist

- [ ] Integration able to create all default resources from scratch
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Resync finishes successfully
- [ ] If new resource kind is added or updated in the integration, add
example raw data, mapping and expected result to the `examples` folder
in the integration directory.
- [ ] If resource kind is updated, run the integration with the example
data and check if the expected result is achieved
- [ ] If new resource kind is added or updated, validate that
live-events for that resource are working as expected
- [ ] Docs PR link [here](#)

### Preflight checklist

- [ ] Handled rate limiting
- [ ] Handled pagination
- [ ] Implemented the code in async
- [ ] Support Multi account

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

Provide links to the API documentation used for this integration.

---------

Co-authored-by: Shalev Avhar <shalev@getport.io>
Co-authored-by: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com>

Author: 3 people

CHANGELOG.md

@@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.14.0 (2024-11-12)
+
+
+### Improvements
+
+- Add support for choosing default resources that the integration will create dynamically
+
 ## 0.13.1 (2024-11-12)
 
 

port_ocean/cli/commands/defaults/clean.py

@@ -51,4 +51,6 @@ def clean(path: str, force: bool, wait: bool) -> None:
         default_app,
     )
 
-    clean_defaults(app.integration.AppConfigHandlerClass.CONFIG_CLASS, force, wait)
+    clean_defaults(
+        app.integration.AppConfigHandlerClass.CONFIG_CLASS, app.config, force, wait
+    )

port_ocean/config/settings.py

@@ -78,6 +78,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
         default_factory=lambda: IntegrationSettings(type="", identifier="")
     )
     runtime: Runtime = Runtime.OnPrem
+    resources_path: str = Field(default=".port/resources")
 
     @root_validator()
     def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:

port_ocean/core/defaults/clean.py

@@ -4,6 +4,7 @@
 import httpx
 from loguru import logger
 
+from port_ocean.config.settings import IntegrationConfiguration
 from port_ocean.context.ocean import ocean
 from port_ocean.core.defaults.common import (
     get_port_integration_defaults,
@@ -14,26 +15,32 @@
 
 def clean_defaults(
     config_class: Type[PortAppConfig],
+    integration_config: IntegrationConfiguration,
     force: bool,
     wait: bool,
 ) -> None:
     try:
         asyncio.new_event_loop().run_until_complete(
-            _clean_defaults(config_class, force, wait)
+            _clean_defaults(config_class, integration_config, force, wait)
         )
 
     except Exception as e:
         logger.error(f"Failed to clear defaults, skipping... Error: {e}")
 
 
 async def _clean_defaults(
-    config_class: Type[PortAppConfig], force: bool, wait: bool
+    config_class: Type[PortAppConfig],
+    integration_config: IntegrationConfiguration,
+    force: bool,
+    wait: bool,
 ) -> None:
     port_client = ocean.port_client
     is_exists = await is_integration_exists(port_client)
     if not is_exists:
         return None
-    defaults = get_port_integration_defaults(config_class)
+    defaults = get_port_integration_defaults(
+        config_class, integration_config.resources_path
+    )
     if not defaults:
         return None
 

port_ocean/core/defaults/common.py

@@ -3,6 +3,7 @@
 from typing import Type, Any, TypedDict, Optional
 
 import httpx
+from loguru import logger
 import yaml
 from pydantic import BaseModel, Field
 from starlette import status
@@ -77,18 +78,33 @@ def deconstruct_blueprints_to_creation_steps(
     )
 
 
+def is_valid_dir(path: Path) -> bool:
+    return path.is_dir()
+
+
 def get_port_integration_defaults(
-    port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".")
+    port_app_config_class: Type[PortAppConfig],
+    custom_defaults_dir: Optional[str] = None,
+    base_path: Path = Path("."),
 ) -> Defaults | None:
-    defaults_dir = base_path / ".port/resources"
-    if not defaults_dir.exists():
-        return None
-
-    if not defaults_dir.is_dir():
-        raise UnsupportedDefaultFileType(
-            f"Defaults directory is not a directory: {defaults_dir}"
+    fallback_dir = base_path / ".port/resources"
+
+    if custom_defaults_dir and is_valid_dir(base_path / custom_defaults_dir):
+        defaults_dir = base_path / custom_defaults_dir
+    elif is_valid_dir(fallback_dir):
+        logger.info(
+            f"Could not find custom defaults directory {custom_defaults_dir}, falling back to {fallback_dir}",
+            fallback_dir=fallback_dir,
+            custom_defaults_dir=custom_defaults_dir,
+        )
+        defaults_dir = fallback_dir
+    else:
+        logger.warning(
+            f"Could not find defaults directory {fallback_dir}, skipping defaults"
         )
+        return None
 
+    logger.info(f"Loading defaults from {defaults_dir}", defaults_dir=defaults_dir)
     default_jsons = {}
     allowed_file_names = [
         field_model.alias for _, field_model in Defaults.__fields__.items()

port_ocean/core/defaults/initialize.py

@@ -198,7 +198,9 @@ async def _initialize_defaults(
     config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     port_client = ocean.port_client
-    defaults = get_port_integration_defaults(config_class)
+    defaults = get_port_integration_defaults(
+        config_class, integration_config.resources_path
+    )
     if not defaults:
         logger.warning("No defaults found. Skipping initialization...")
         return None

port_ocean/tests/core/defaults/test_common.py

@@ -0,0 +1,166 @@
+import pytest
+import json
+from unittest.mock import patch
+from pathlib import Path
+from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.core.defaults.common import (
+    get_port_integration_defaults,
+    Defaults,
+)
+
+
+@pytest.fixture
+def setup_mock_directories(tmp_path: Path) -> tuple[Path, Path, Path]:
+    # Create .port/resources with sample files
+    default_dir = tmp_path / ".port/resources"
+    default_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create mock JSON and YAML files with expected content
+    (default_dir / "blueprints.json").write_text(
+        json.dumps(
+            [
+                {
+                    "identifier": "mock-identifier",
+                    "title": "mock-title",
+                    "icon": "mock-icon",
+                    "schema": {
+                        "type": "object",
+                        "properties": {"key": {"type": "string"}},
+                    },
+                }
+            ]
+        )
+    )
+    (default_dir / "port-app-config.json").write_text(
+        json.dumps(
+            {
+                "resources": [
+                    {
+                        "kind": "mock-kind",
+                        "selector": {"query": "true"},
+                        "port": {
+                            "entity": {
+                                "mappings": {
+                                    "identifier": ".id",
+                                    "title": ".title",
+                                    "blueprint": '"mock-identifier"',
+                                }
+                            }
+                        },
+                    }
+                ]
+            }
+        )
+    )
+
+    # Create .port/custom_resources with different sample files
+    custom_resources_dir = tmp_path / ".port/custom_resources"
+    custom_resources_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create mock JSON and YAML files with expected content
+    (custom_resources_dir / "blueprints.json").write_text(
+        json.dumps(
+            [
+                {
+                    "identifier": "mock-custom-identifier",
+                    "title": "mock-custom-title",
+                    "icon": "mock-custom-icon",
+                    "schema": {
+                        "type": "object",
+                        "properties": {"key": {"type": "string"}},
+                    },
+                }
+            ]
+        )
+    )
+    (custom_resources_dir / "port-app-config.json").write_text(
+        json.dumps(
+            {
+                "resources": [
+                    {
+                        "kind": "mock-custom-kind",
+                        "selector": {"query": "true"},
+                        "port": {
+                            "entity": {
+                                "mappings": {
+                                    "identifier": ".id",
+                                    "title": ".title",
+                                    "blueprint": '"mock-custom-identifier"',
+                                }
+                            }
+                        },
+                    }
+                ]
+            }
+        )
+    )
+
+    # Define the non-existing directory path
+    non_existing_dir = tmp_path / ".port/do_not_exist"
+
+    return default_dir, custom_resources_dir, non_existing_dir
+
+
+def test_custom_defaults_dir_used_if_valid(
+    setup_mock_directories: tuple[Path, Path, Path]
+) -> None:
+    # Arrange
+    _, custom_resources_dir, _ = setup_mock_directories
+
+    with (
+        patch("port_ocean.core.defaults.common.is_valid_dir") as mock_is_valid_dir,
+        patch(
+            "pathlib.Path.iterdir",
+            return_value=custom_resources_dir.iterdir(),
+        ),
+    ):
+        mock_is_valid_dir.side_effect = lambda path: path == custom_resources_dir
+
+        # Act
+        defaults = get_port_integration_defaults(
+            port_app_config_class=PortAppConfig,
+            custom_defaults_dir=".port/custom_resources",
+            base_path=custom_resources_dir.parent.parent,
+        )
+
+        # Assert
+        assert isinstance(defaults, Defaults)
+        assert defaults.blueprints[0].get("identifier") == "mock-custom-identifier"
+        assert defaults.port_app_config is not None
+        assert defaults.port_app_config.resources[0].kind == "mock-custom-kind"
+
+
+def test_fallback_to_default_dir_if_custom_dir_invalid(
+    setup_mock_directories: tuple[Path, Path, Path]
+) -> None:
+    resources_dir, _, non_existing_dir = setup_mock_directories
+
+    # Arrange
+    with (
+        patch("port_ocean.core.defaults.common.is_valid_dir") as mock_is_valid_dir,
+        patch("pathlib.Path.iterdir", return_value=resources_dir.iterdir()),
+    ):
+
+        mock_is_valid_dir.side_effect = lambda path: path == resources_dir
+
+        # Act
+        custom_defaults_dir = str(non_existing_dir.relative_to(resources_dir.parent))
+        defaults = get_port_integration_defaults(
+            port_app_config_class=PortAppConfig,
+            custom_defaults_dir=custom_defaults_dir,
+            base_path=resources_dir.parent.parent,
+        )
+
+        # Assert
+        assert isinstance(defaults, Defaults)
+        assert defaults.blueprints[0].get("identifier") == "mock-identifier"
+        assert defaults.port_app_config is not None
+        assert defaults.port_app_config.resources[0].kind == "mock-kind"
+
+
+def test_default_resources_path_does_not_exist() -> None:
+    # Act
+    defaults = get_port_integration_defaults(port_app_config_class=PortAppConfig)
+
+    # Assert
+    assert defaults is None

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.13.1"
+version = "0.14.0"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"