Feature/netsuite2 fiscal year end

.buildkite/scripts/run_models.sh

@@ -22,15 +22,15 @@ dbt seed --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --t
 dbt compile --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
 dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false, netsuite2__fiscal_calendar_enabled: true}' --target "$db" --full-refresh
 dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
 
 else
 
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false, netsuite2__fiscal_calendar_enabled: true}' --target "$db" --full-refresh
 dbt test --target "$db"
 fi
 

CHANGELOG.md

@@ -1,4 +1,9 @@
-# dbt_netsuite_source version.version
+# dbt_netsuite_source v0.12.0
+
+## Breaking Schema Changes
+- Added optional `fiscalcalendar` source table to support accurate fiscal year start dates (currently defaulted to calendar year). This table and related models (`stg_netsuite2__fiscal_calendar_tmp` and `stg_netsuite2__fiscal_calendar`) are disabled by default.
+  - To enable: turn on the fiscalcalendar table in the connection schema tab.
+  - For dbt Core, also set the `netsuite2__fiscal_calendar_enabled` variable to true (default is false).
 
 ## Documentation
 - Corrected references to connectors and connections in the README. ([#60](https://github.com/fivetran/dbt_netsuite_source/pull/60))

README.md

@@ -63,6 +63,7 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite**
 - employee
 - entity
 - entityaddress
+- fiscalcalendar (required for non–January 1 fiscal year start)
 - item
 - job
 - location
@@ -91,7 +92,7 @@ If you  are **not** using the [Netsuite transformation package](https://github.c
 ```yaml
 packages:
   - package: fivetran/netsuite_source
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 ```
 
 ### Step 3: Define Netsuite.com or Netsuite2 Source
@@ -111,14 +112,15 @@ vars:
 ```
 
 ### Step 5: Disable models for non-existent sources (Netsuite2 only)
-It's possible that your Netsuite connection does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, all variables are assumed to be true. Add variables for only the tables you would like to disable:
+It's possible that your Netsuite connection does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, most variables are assumed to be true with the exception of `netsuite2__fiscal_calendar_enabled`. Add variables for only the tables you would like to disable/enable respectively:
 ```yml
 vars:
     netsuite2__multibook_accounting_enabled: false # True by default. Disable `accountingbooksubsidiary` and `accountingbook` if you are not using the Multi-Book Accounting feature
     netsuite2__using_exchange_rate: false #True by default. Disable `exchange_rate` if you don't utilize exchange rates. If you set this variable to false and are using the `netsuite` transformation package, ensure it is scoped globally so that the `netsuite` package can access it as well.  
     netsuite2__using_vendor_categories: false # True by default. Disable `vendorcategory` if you don't categorize your vendors
     netsuite2__using_jobs: false # True by default. Disable `job` if you don't use jobs
     netsuite2__using_employees: false # True by default. Disable `employee` if you don't use employees.
+    netsuite2__fiscal_calendar_enabled: true # False by default. Enable `fiscalcalendar` if you have a fiscal year starting on a month different than January.
 
 ```
 

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.11.1'
+version: '0.12.0'
 
 models:
   netsuite_source:
@@ -54,6 +54,7 @@ vars:
     netsuite2_vendor_categories: "{{ source('netsuite2', 'vendor_category') }}"
     netsuite2_accounting_book_subsidiaries: "{{ source('netsuite2', 'accounting_book_subsidiaries') }}"
     netsuite2_accounting_period_fiscal_calendars: "{{ source('netsuite2', 'accounting_period_fiscal_calendars') }}"
+    netsuite2_fiscal_calendar: "{{ source('netsuite2', 'fiscal_calendar') }}"
     netsuite2_account_types: "{{ source('netsuite2', 'account_type') }}"
     netsuite2_entity_address: "{{ source('netsuite2', 'entity_address') }}"
     netsuite2_location_main_address: "{{ source('netsuite2', 'location_main_address') }}"

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.11.1'
+version: '0.12.0'
 
 profile: 'integration_tests'
 config-version: 2
@@ -47,6 +47,7 @@ vars:
     netsuite2_vendor_category_identifier: "netsuite2_vendor_category_data"
     netsuite2_accounting_book_subsidiaries_identifier: "netsuite2_accounting_book_sub_data"
     netsuite2_accounting_period_fiscal_calendars_identifier: "netsuite2_accounting_period_fiscal_cal_data"
+    netsuite2_fiscal_calendar_identifier: "netsuite2_fiscal_calendar_data"
     netsuite2_account_type_identifier: "netsuite2_account_type_data"
     netsuite2_entity_address_identifier: "netsuite2_entity_address_data"
     netsuite2_location_main_address_identifier: "netsuite2_location_main_address_data"

integration_tests/seeds/netsuite2_accounting_period_fiscal_cal_data.csv

@@ -1,7 +1,7 @@
 _fivetran_id,_fivetran_deleted,_fivetran_synced,accountingperiod,date_deleted,fiscalcalendar,parent
 t6+LTClexXfChh/pMhoj1F9SpSs=,FALSE,2022-07-10 22:17:10,245,,1,244
 hbwug8y6jPTTembEaYVZlsljZuQ=,FALSE,2022-07-10 22:17:10,252,,1,239
-hxkIkIIeDjsG8OgPAlvQVe9a6TE=,FALSE,2022-07-10 22:17:10,275,,1,274
-H00Ert0t6kQHovZX4pHHC4N4d0Y=,FALSE,2022-07-10 22:17:10,6,,1,1
-KI+pra1uPLFPsBHFuvT/tXCaWdg=,FALSE,2022-07-10 22:17:10,184,,1,171
-JIpeyH5xsYzCF0YbPKr7obXrkhA=,FALSE,2022-07-10 22:17:10,218,,1,205
+hxkIkIIeDjsG8OgPAlvQVe9a6TE=,FALSE,2022-07-10 22:17:10,275,,2,274
+H00Ert0t6kQHovZX4pHHC4N4d0Y=,FALSE,2022-07-10 22:17:10,6,,2,1
+KI+pra1uPLFPsBHFuvT/tXCaWdg=,FALSE,2022-07-10 22:17:10,20,,3,171
+JIpeyH5xsYzCF0YbPKr7obXrkhA=,FALSE,2022-07-10 22:17:10,21,,3,205

integration_tests/seeds/netsuite2_fiscal_calendar_data.csv

@@ -0,0 +1,4 @@
+id,externalid,fiscalmonth,isdefault,name,_fivetran_deleted,date_deleted,_fivetran_synced
+1,,JAN,true,Calendar Year,false,,2022-05-27 13:00:58
+2,,APR,false,April Fiscal Year,false,,2022-05-27 13:00:58
+3,,CAL,false,Incorrectly Formed Month Abbreviation,false,,2022-05-27 13:00:58

macros/get_fiscalcalendar_columns.sql

@@ -0,0 +1,16 @@
+{% macro get_fiscalcalendar_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "externalid", "datatype": dbt.type_string()},
+    {"name": "fiscalmonth", "datatype": dbt.type_string()},
+    {"name": "isdefault", "datatype": dbt.type_string()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "date_deleted", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/netsuite2/src_netsuite2.yml

@@ -91,6 +91,29 @@ sources:
           - name: parent
             description: Reference to the parent fiscal calendar accounting period.
 
+      - name: fiscal_calendar
+        identifier: "{{ var('netsuite2_fiscal_calendar_identifier', 'fiscalcalendar') }}"
+        description: Table containing a list of fiscal calendars, their names, and starting fiscal months. Used to determine fiscal year boundaries.
+        config:
+          enabled: "{{ var('netsuite2__fiscal_calendar_enabled', false) }}"
+        columns:
+          - name: id
+            description: Internal identifier for the fiscal calendar record.
+          - name: externalid
+            description: Custom or system-defined external identifier for the fiscal calendar.
+          - name: fiscalmonth
+            description: The starting month of the fiscal calendar (e.g., JAN for January, APR for April).
+          - name: isdefault
+            description: Boolean flag indicating whether this is the default fiscal calendar.
+          - name: name
+            description: Name of the fiscal calendar (e.g., 'Standard Fiscal Year').
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
+          - name: date_deleted
+            description: Timestamp when the record was deleted in the source system, if applicable.
+          - name: _fivetran_synced
+            description: "{{ doc('_fivetran_synced') }}"
+
       - name: accounting_period
         identifier: "{{ var('netsuite2_accounting_period_identifier', 'accountingperiod') }}"
         description: "{{ doc('account_periods_table') }}"

models/netsuite2/stg_netsuite2__fiscal_calendar.sql

@@ -0,0 +1,37 @@
+{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__fiscal_calendar_enabled', false)) }}
+
+with base as (
+
+    select * 
+    from {{ ref('stg_netsuite2__fiscal_calendar_tmp') }}
+),
+
+fields as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_netsuite2__fiscal_calendar_tmp')),
+                staging_columns=get_fiscalcalendar_columns()
+            )
+        }}
+    from base
+),
+
+final as (
+
+    select 
+        id as fiscal_calendar_id,
+        externalid as external_id,
+        fiscalmonth as fiscal_month,
+        isdefault as is_default,
+        name,
+        date_deleted,
+        _fivetran_deleted,
+        _fivetran_synced
+    from fields
+    where not coalesce(_fivetran_deleted, false)
+)
+
+select *
+from final

models/netsuite2/tmp/stg_netsuite2__fiscal_calendar_tmp.sql

@@ -0,0 +1,4 @@
+{{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__fiscal_calendar_enabled', false)) }}
+
+select * 
+from {{ var('netsuite2_fiscal_calendar') }}

models/stg_netsuite2.yml

@@ -81,6 +81,29 @@ models:
       - name: parent_id
         description: Reference to the parent fiscal calendar accounting period.
 
+  - name: stg_netsuite2__fiscal_calendar
+    description: Table containing a list of fiscal calendars, their names, and starting fiscal months. Used to determine fiscal year boundaries.
+    columns:
+      - name: fiscal_calendar_id
+        description: Internal identifier for the fiscal calendar record.
+        tests: 
+          - unique
+          - not_null
+      - name: external_id
+        description: Custom or system-defined external identifier for the fiscal calendar.
+      - name: fiscal_month
+        description: The starting month of the fiscal calendar (e.g., JAN for January, APR for April).
+      - name: is_default
+        description: "Boolean flag indicating whether this is the default fiscal calendar.
+      - name: name
+        description: Name of the fiscal calendar (e.g., 'Standard Fiscal Year')."
+      - name: _fivetran_deleted
+        description: "{{ doc('_fivetran_deleted') }}"
+      - name: date_deleted
+        description: Timestamp when the record was deleted in the source system, if applicable.
+      - name: _fivetran_synced
+        description: "{{ doc('_fivetran_synced') }}"
+
   - name: stg_netsuite2__accounting_periods
     description: "{{ doc('account_periods_table') }}"
     columns: