Merge pull request #7 from fivetran/bug/multi-fixes

fivetran-reneeli · web-flow · commit 1a42b0da3138 · 2022-04-21T16:30:04.000-05:00
Bug/multi fixes
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -22,26 +22,19 @@ jobs:
             pip install -r integration_tests/requirements.txt
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
-      - run:
-          name: "Run Tests - Spark"
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps
-            dbt seed --target spark --full-refresh
-            dbt run --target spark --full-refresh
-            dbt test --target spark
       - run:
           name: "Run Tests - Redshift"
           command: |
             . venv/bin/activate
             echo `pwd`
             cd integration_tests
+            dbt clean
             dbt deps
             dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
             dbt test --target redshift
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target redshift
+            dbt test --target redshift
       - run:
           name: "Run Tests - Postgres"
           command: |
@@ -52,6 +45,20 @@ jobs:
             dbt seed --target postgres --full-refresh
             dbt run --target postgres --full-refresh
             dbt test --target postgres
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target postgres
+            dbt test --target postgres
+      - run:
+          name: "Run Tests - Spark"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target spark --full-refresh
+            dbt run --target spark --full-refresh
+            dbt test --target spark
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target spark
+            dbt test --target spark
       - run:
           name: "Run Tests - Snowflake"
           command: |
@@ -62,6 +69,8 @@ jobs:
             dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
             dbt test --target snowflake
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target snowflake
+            dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
           environment:
@@ -74,4 +83,6 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
+            dbt test --target bigquery
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target bigquery
             dbt test --target bigquery
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,24 @@
+# dbt_sage_intacct_source v0.1.1
+ 
+## Updates
+- Excludes fivetran-deleted rows in the staging models
+([#7](https://github.com/fivetran/dbt_sage_intacct/issues/7))
+
+- Allow for pass-through columns from the  `gl_detail` and `gl_account` source tables using the following variables. For more information please refer to the [README](https://github.com/fivetran/dbt_sage_intacct_source/blob/main/README.md#passthrough-columns)
+    - `sage_account_pass_through_columns`
+    - `sage_gl_pass_through_columns`
+
+- Add variable configs for the AP Bill and AR Invoice related tables to allow users to disable them if not used. For more information please refer to the [README](https://github.com/fivetran/dbt_sage_intacct_source/blob/main/README.md#disabling-and-enabling-models)
+    - `sage_intacct__using_bills`
+    - `sage_intacct__using_invoices`
+
+- Include additional fields in the `gl_detail` model: customer_id, customer_name, department_id, department_title, location_id, location_name, vendor_id, and vendor_name. 
+
+([#6](https://github.com/fivetran/dbt_sage_intacct/issues/6))
+
+## Contributors
+Thank you @santi95 for raising these to our attention! ([#6](https://github.com/fivetran/dbt_sage_intacct_source/pull/6))
+
 # dbt_sage_intacct_source v0.1.0
 
 ## Initial Release
diff --git a/README.md b/README.md
@@ -56,8 +56,26 @@ Columns passed through must be present in the upstream source tables. See below
 vars:
   sage_intacct_source:
     sage_account_pass_through_columns: ['new_custom_field', 'custom_field_2']
+    sage_gl_pass_through_columns: ['custom_field_3', 'custom_field_4']
 ```
 
+### Disabling and Enabling Models
+
+When setting up your Sage Intacct connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Sage or have actively decided to not sync some tables. In order to disable the relevant functionality in the package, you will need to add the relevant variables.
+
+By default, all variables are assumed to be `true`. You only need to add variables for the tables you would like to disable:
+
+```yml
+# dbt_project.yml
+
+config-version: 2
+
+vars:
+    sage_intacct__using_invoices: false                 # default is true
+    sage_intacct__using_bills: false                    # default is true
+```
+
+
 ### Change the Build Schema
 By default, this package builds the Sage Intacct staging models within a schema titled (<target_schema> + `_stg_sage_intacct`) in your target database. If this is not where you would like your Sage Intacct staging data to be written to, add the following configuration to your `dbt_project.yml` file:
 
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'sage_intacct_source'
-version: '0.1.0'
+version: '0.1.1'
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 
@@ -19,5 +19,14 @@ vars:
     ar_invoice_item:              "{{ source('sage_intacct','ar_invoice_item') }}"
     ar_invoice:                   "{{ source('sage_intacct','ar_invoice') }}"
     
-    sage_account_pass_through_columns: [] 
+    sage_account_pass_through_columns: []
+    sage_gl_pass_through_columns: []
 
+    sage_intacct__using_bills: true
+    sage_intacct__using_invoices: true 
+    
+    
+
+clean-targets:
+    - target
+    - dbt_modules
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,6 +1,6 @@
 
 name: 'sage_intacct_source_integration_tests'
-version: '0.1.0'
+version: '0.1.1'
 config-version: 2
 
 profile: 'integration_tests' 
diff --git a/macros/get_gl_detail_columns.sql b/macros/get_gl_detail_columns.sql
@@ -1,5 +1,4 @@
 {% macro get_gl_detail_columns() %}
-
 {% set columns = [
     {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "_fivetran_synced", "datatype": dbt_utils.type_timestamp()},
@@ -12,11 +11,15 @@
     {"name": "batch_title", "datatype": dbt_utils.type_string()},
     {"name": "batchkey", "datatype": dbt_utils.type_string()},
     {"name": "bookid", "datatype": dbt_utils.type_string()},
+    {"name": "classid", "datatype": dbt_utils.type_string()},
+    {"name": "classname", "datatype": dbt_utils.type_string()},
     {"name": "creditamount", "datatype": dbt_utils.type_float()},
     {"name": "currency", "datatype": dbt_utils.type_string()},
     {"name": "customerid", "datatype": dbt_utils.type_string()},
     {"name": "customername", "datatype": dbt_utils.type_string()},
     {"name": "debitamount", "datatype": dbt_utils.type_float()},
+    {"name": "departmentid", "datatype": dbt_utils.type_string()},
+    {"name": "departmenttitle", "datatype": dbt_utils.type_string()},
     {"name": "description", "datatype": dbt_utils.type_string()},
     {"name": "docnumber", "datatype": dbt_utils.type_string()},
     {"name": "document", "datatype": dbt_utils.type_string()},
@@ -27,6 +30,7 @@
     {"name": "locationid", "datatype": dbt_utils.type_string()},
     {"name": "locationname", "datatype": dbt_utils.type_string()},
     {"name": "modified", "datatype": dbt_utils.type_timestamp()},
+    {"name": "prdescription", "datatype": dbt_utils.type_string()},
     {"name": "recordid", "datatype": dbt_utils.type_string()},
     {"name": "recordno", "datatype": dbt_utils.type_string()},
     {"name": "recordtype", "datatype": dbt_utils.type_string()},
@@ -44,7 +48,5 @@
     {"name": "whenmodified", "datatype": dbt_utils.type_timestamp()},
     {"name": "whenpaid", "datatype": "date"}
 ] %}
-
 {{ return(columns) }}
-
-{% endmacro %}
+{% endmacro %}
diff --git a/models/stg_sage_intacct__ap_bill.sql b/models/stg_sage_intacct__ap_bill.sql
@@ -1,3 +1,4 @@
+{{ config(enabled=var('sage_intacct__using_bills', True)) }}
 
 with base as (
 
@@ -52,3 +53,4 @@ final as (
 )
 
 select * from final
+where not coalesce(_fivetran_deleted, false)
diff --git a/models/stg_sage_intacct__ap_bill_item.sql b/models/stg_sage_intacct__ap_bill_item.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=var('sage_intacct__using_bills', True)) }}
+
 with base as (
 
     select * 
diff --git a/models/stg_sage_intacct__ar_invoice.sql b/models/stg_sage_intacct__ar_invoice.sql
@@ -1,3 +1,4 @@
+{{ config(enabled=var('sage_intacct__using_invoices', True)) }}
 
 with base as (
 
@@ -55,3 +56,4 @@ final as (
 )
 
 select * from final
+where not coalesce(_fivetran_deleted, false)
diff --git a/models/stg_sage_intacct__ar_invoice_item.sql b/models/stg_sage_intacct__ar_invoice_item.sql
@@ -1,3 +1,4 @@
+{{ config(enabled=var('sage_intacct__using_invoices', True)) }}
 
 with base as (
 
diff --git a/models/stg_sage_intacct__gl_account.sql b/models/stg_sage_intacct__gl_account.sql
@@ -60,3 +60,4 @@ final as (
 )
 
 select * from final
+where not coalesce(_fivetran_deleted, false)
diff --git a/models/stg_sage_intacct__gl_detail.sql b/models/stg_sage_intacct__gl_detail.sql
@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_gl_detail_columns()
             )
         }}
+        --The below script allows for pass through columns.
+        {% if var('sage_gl_pass_through_columns') %} 
+        ,
+        {{ var('sage_gl_pass_through_columns') | join (", ")}}
+
+        {% endif %}
         
     from base
 ),
@@ -23,6 +29,7 @@ final as (
     
     select 
         recordno as gl_detail_id,
+        _fivetran_deleted,
         cast(accountno as {{ dbt_utils.type_string() }}) as account_no,
         accounttitle as account_title,
         amount,
@@ -31,17 +38,24 @@ final as (
         batch_title,
         batchkey as batch_key,
         bookid as book_id,
+        classid as class_id,
+        classname as class_name,
         creditamount as credit_amount,
         debitamount as debit_amount,
         currency,
         customerid as customer_id,
         customername as customer_name,
+        departmentid as department_id,
+        departmenttitle as department_title,
         description,
         docnumber as doc_number,
         entry_date as entry_date_at,
         entry_state,
         entrydescription as entry_description,
         line_no,
+        locationid as location_id,
+        locationname as location_name,
+        prdescription as pr_description,
         recordid as record_id,
         recordtype as record_type,
         totaldue as total_due,
@@ -58,7 +72,16 @@ final as (
         whenmodified as modified_at,
         whenpaid as paid_at
 
+
+        --The below script allows for pass through columns.
+        {% if var('sage_gl_pass_through_columns') %} 
+        ,
+        {{ var('sage_gl_pass_through_columns') | join (", ")}}
+
+        {% endif %}
+
     from fields
 )
 
 select * from final
+where not coalesce(_fivetran_deleted, false)
diff --git a/models/tmp/stg_sage_intacct__ar_invoice_item_tmp.sql b/models/tmp/stg_sage_intacct__ar_invoice_item_tmp.sql
@@ -1 +1,3 @@
+{{ config(enabled=var('sage_intacct__using_invoices', True)) }}
+
 select * from {{ var('ar_invoice_item') }}
diff --git a/models/tmp/stg_sage_intacct__ar_invoice_tmp.sql b/models/tmp/stg_sage_intacct__ar_invoice_tmp.sql
@@ -1 +1,3 @@
+{{ config(enabled=var('sage_intacct__using_invoices', True)) }}
+
 select * from {{ var('ar_invoice') }}

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+{{ config(enabled=var('sage_intacct__using_bills', True)) }}`
`1`	`2`
`2`	`3`	`with base as (`
`3`	`4`
`@@ -52,3 +53,4 @@ final as (`
`52`	`53`	`)`
`53`	`54`
`54`	`55`	`select * from final`
	`56`	`+where not coalesce(_fivetran_deleted, false)`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+{{ config(enabled=var('sage_intacct__using_bills', True)) }}`
	`2`	`+`
`1`	`3`	`with base as (`
`2`	`4`
`3`	`5`	`select *`