Fix-input_gcp_projects (#139)

* fix input_gcp_projects

* Refactor project_list macro to improve handling of string inputs and array parsing. Simplified logic for processing project strings and enhanced handling of empty arrays.

* Add documentation

* Add changie

* Use project_by_project_view for integration testing

* add exclude to organization models as not available in test env

---------

Co-authored-by: Christophe Oudar <kayrnt@gmail.com>

Author: github-christophe-oudar

.changes/unreleased/Fixes-20250719-214703.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix input_gcp_projects parsing and avoid fromjson for dbt_fusion compatibility
+time: 2025-07-19T21:47:03.320443+02:00
+custom:
+    Author: Kayrnt
+    Issue: ""

.github/workflows/pr_run_models.yml

@@ -17,8 +17,9 @@ env:
   DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT }}
-  DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION }}
+  DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION: "us"
   DBT_BQ_MONITORING_GCP_PROJECTS: ${{ secrets.DBT_BQ_MONITORING_GCP_PROJECTS }}
+  DBT_BQ_MONITORING_GOOGLE_INFORMATION_SCHEMA_MODELS_MATERIALIZATION: project_by_project_view
 
 concurrency:
   cancel-in-progress: true
@@ -28,6 +29,9 @@ jobs:
   integration-bigquery:
     name: Run dbt models on BigQuery to test model validity
     runs-on: ubuntu-latest
+    
+    env:
+      DBT_COMMON_FLAGS: "--empty --exclude information_schema_streaming_timeline_by_organization information_schema_jobs_timeline_by_organization information_schema_table_storage_by_organization information_schema_table_storage_by_organization information_schema_write_api_timeline_by_organization information_schema_jobs_by_organization information_schema_table_storage_usage_timeline_by_organization"
 
     steps:
       - name: Checkout
@@ -64,34 +68,34 @@ jobs:
       - name: Run all models once
         run: | 
           cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --full-refresh --empty
+          dbt build -s dbt_bigquery_monitoring --full-refresh ${{ env.DBT_COMMON_FLAGS }}
 
       - name: Run all models again to test incremental
         run: |       
           cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --empty
+          dbt build -s dbt_bigquery_monitoring ${{ env.DBT_COMMON_FLAGS }}
 
       - name: Run all models again with cloud audit logs
         run: |       
           cd integration_tests
           DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --full-refresh --empty
+          dbt run -s jobs_from_audit_logs+ --full-refresh ${{ env.DBT_COMMON_FLAGS }}
 
       - name: Run all models again to test incremental with cloud audit logs
         run: |       
           cd integration_tests
           DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --empty
+          dbt run -s jobs_from_audit_logs+ ${{ env.DBT_COMMON_FLAGS }}
 
       # Disable until billing is enabled as DML is not available in the free tier
       # - name: Run all models again in project mode
       #   run: |       
       #     cd integration_tests
       #     DBT_BQ_MONITORING_GCP_PROJECTS="['${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}']" \
-      #     dbt build -s dbt_bigquery_monitoring --full-refresh --empty
+      #     dbt build -s dbt_bigquery_monitoring --full-refresh ${{ env.DBT_COMMON_FLAGS }}
 
       # - name: Run all models again in project mode
       #   run: |       
       #     cd integration_tests
       #     DBT_BQ_MONITORING_GCP_PROJECTS="['${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}']" \
-      #     dbt build -s dbt_bigquery_monitoring --empty
+      #     dbt build -s dbt_bigquery_monitoring ${{ env.DBT_COMMON_FLAGS }}

docs/configuration/configuration.md

@@ -54,6 +54,44 @@ vars:
   input_gcp_projects: [ 'my-gcp-project', 'my-gcp-project-2' ]
 ```
 
+#### Supported Input Formats
+
+The `input_gcp_projects` setting accepts multiple input formats for maximum flexibility:
+
+**1. dbt project variables (dbt_project.yml):**
+```yml
+vars:
+  input_gcp_projects: "single-project"                    # Single project as string
+  input_gcp_projects: ["project1", "project2"]            # Multiple projects as array
+```
+
+**2. CLI variables:**
+```bash
+dbt run --vars '{"input_gcp_projects": "test"}'                   # Single project
+dbt run --vars '{"input_gcp_projects": ["test1", "test2"]}'       # Multiple projects
+```
+
+**3. Environment variables:**
+```bash
+# Single project
+export DBT_BQ_MONITORING_GCP_PROJECTS="single-project"
+
+# Multiple projects with quotes
+export DBT_BQ_MONITORING_GCP_PROJECTS='["project1","project2"]'
+
+# Multiple projects without quotes (also supported)
+export DBT_BQ_MONITORING_GCP_PROJECTS='[project1,project2]'
+
+# Single project in array format
+export DBT_BQ_MONITORING_GCP_PROJECTS='["project1"]'
+export DBT_BQ_MONITORING_GCP_PROJECTS='[project1]'
+
+# Empty array
+export DBT_BQ_MONITORING_GCP_PROJECTS='[]'
+```
+
+All input formats are automatically normalized to an array of project strings internally, so you can use whichever format is most convenient for your setup.
+
 :::warning
 
 When using the "project mode", the package will create intermediate tables to avoid issues from BigQuery when too many projects are used.

macros/project_list.sql

@@ -1,18 +1,90 @@
-{#-- retrieve the projects list --#}
+{#--
+  Retrieve the projects list from various input formats and always return an array.
+
+  This macro supports multiple input formats and normalizes them to a consistent array output:
+
+  SUPPORTED INPUT FORMATS:
+
+  1. dbt project variables (dbt_project.yml):
+     vars:
+       input_gcp_projects: "single-project"                    # → ["single-project"]
+       input_gcp_projects: ["project1", "project2"]            # → ["project1", "project2"]
+
+  2. CLI variables:
+     --vars '{"input_gcp_projects": "test"}'                   # → ["test"]
+     --vars '{"input_gcp_projects": ["test1", "test2"]}'       # → ["test1", "test2"]
+
+  3. Environment variables:
+     DBT_BQ_MONITORING_GCP_PROJECTS="single-project"          # → ["single-project"]
+     DBT_BQ_MONITORING_GCP_PROJECTS='["project1","project2"]' # → ["project1", "project2"]
+     DBT_BQ_MONITORING_GCP_PROJECTS='[project1,project2]'     # → ["project1", "project2"] (unquoted)
+     DBT_BQ_MONITORING_GCP_PROJECTS='["project1"]'            # → ["project1"]
+     DBT_BQ_MONITORING_GCP_PROJECTS='[project1]'              # → ["project1"] (unquoted)
+     DBT_BQ_MONITORING_GCP_PROJECTS='[]'                      # → []
+
+  4. Mixed quote styles:
+     DBT_BQ_MONITORING_GCP_PROJECTS="['proj1',\"proj2\"]"     # → ["proj1", "proj2"]
+
+  EDGE CASES HANDLED:
+  - Empty strings return empty arrays
+  - Whitespace around project names is automatically trimmed
+  - Surrounding quotes (single or double) are automatically removed
+  - Invalid/malformed inputs fallback gracefully
+
+  RETURN VALUE:
+  Always returns an array of project strings
+--#}
 {% macro project_list() %}
   {% set projects = dbt_bigquery_monitoring_variable_input_gcp_projects() %}
+
+  {#-- If it's already a list/array, return it directly --#}
   {% if projects is iterable and projects is not string %}
-     {{ return(projects) }}
-  {#-- check if it's the string and it contains a "," --#}
+    {{ return(projects) }}
+
+  {#-- Handle string inputs --#}
   {% elif projects is string %}
-     {% if projects == '' %}
-       {{ return([]) }}
-     {% else %}
-       {% set projects_replaced = projects | replace("'", '"') %}
-       {% set json = fromjson('{"v":' ~ projects_replaced ~ '}') %}
-       {{ return (json['v']) }}
-     {% endif %}
+    {#-- Empty string case --#}
+    {% if projects == '' %}
+      {{ return([]) }}
+    {% endif %}
+
+    {#-- Check if it looks like an array (starts with [ and ends with ]) --#}
+    {% if projects.startswith('[') and projects.endswith(']') %}
+      {#-- Extract content between brackets --#}
+      {% set inner_content = projects[1:-1].strip() %}
+
+      {#-- Handle empty array --#}
+      {% if inner_content == '' %}
+        {{ return([]) }}
+      {% endif %}
+
+      {#-- Split by comma and process each project --#}
+      {% set project_list = [] %}
+      {% set raw_projects = inner_content.split(',') %}
+
+      {% for raw_project in raw_projects %}
+        {% set cleaned_project = raw_project.strip() %}
+
+        {#-- Remove surrounding quotes if present --#}
+        {% if (cleaned_project.startswith('"') and cleaned_project.endswith('"')) or
+              (cleaned_project.startswith("'") and cleaned_project.endswith("'")) %}
+          {% set cleaned_project = cleaned_project[1:-1] %}
+        {% endif %}
+
+        {#-- Add non-empty projects to the list --#}
+        {% if cleaned_project %}
+          {% set _ = project_list.append(cleaned_project) %}
+        {% endif %}
+      {% endfor %}
+
+      {{ return(project_list) }}
+    {% else %}
+      {#-- Single project string - wrap in array --#}
+      {{ return([projects]) }}
+    {% endif %}
+
+  {#-- Fallback: return empty array for any other type --#}
   {% else %}
-    {{ exceptions.raise_compiler_error('Invalid `input_gcp_projects` variables. Got: ' ~ input_gcp_projects ~ ' but expected form like input_gcp_projects = ["project_1", "project_2"] ') }}
+    {{ return([]) }}
   {% endif %}
 {% endmacro %}