diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4c799f0..dc3ba52 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -18,7 +18,7 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -35,7 +35,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -47,7 +47,7 @@ steps: key: "run_dbt_redshift" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -62,7 +62,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 554a2a4..c75e79d 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -17,6 +17,7 @@ echo `pwd` cd integration_tests dbt deps dbt seed --target "$db" --full-refresh +dbt source freshness --target "$db" || echo "...Only verifying freshness runs…" dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target "$db" diff --git a/CHANGELOG.md b/CHANGELOG.md index 2057f57..535d8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +# dbt_sage_intacct_source v0.5.0 + +[PR #23](https://github.com/fivetran/dbt_sage_intacct_source/pull/23) includes the following updates: + +## Breaking Change for dbt Core < 1.9.6 + +> *Note: This is not relevant to Fivetran Quickstart users.* + +Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core. This will resolve the following deprecation warning that users running dbt >= 1.9.6 may have received: + +``` +[WARNING]: Deprecated functionality +Found `freshness` as a top-level property of `tiktok_ads` in file +`models/src_tiktok_ads.yml`. The `freshness` top-level property should be moved +into the `config` of `tiktok_ads`. +``` + +**IMPORTANT:** Users running dbt Core < 1.9.6 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests. + +If you are using dbt Core < 1.9.6 and want to continue running Sage Intacct freshness tests, please elect **one** of the following options: + 1. (Recommended) Upgrade to dbt Core >= 1.9.6 + 2. Do not upgrade your installed version of the `sage_intacct_source` package. Pin your dependency on v0.4.1 in your `packages.yml` file. + 3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `sage_intacct` source and apply freshness via the previous release top-level property route. This will require you to copy and paste the entirety of the previous release `src_sage_intacct.yml` file and add an `overrides: sage_intacct_source` property. + +## Under the Hood +- Updates to ensure integration tests use latest version of dbt. + # dbt_sage_intacct_source v0.4.1 This release includes the following updates: diff --git a/README.md b/README.md index 78a8611..10d882a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - Materializes [Sage Intacct staging tables](https://fivetran.github.io/dbt_sage_intacct_source/#!/overview/sage_intacct_source/models/?g_v=1&g_e=seeds) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/sage-intacct#schemainformation). These staging tables clean, test, and prepare your Sage Intacct data from [Fivetran's connector](https://fivetran.com/docs/applications/sage-intacct) for analysis by doing the following: - Names columns for consistency across all packages and for easier analysis - Adds freshness tests to source data + - dbt Core >= 1.9.6 is required to run freshness tests out of the box. - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values. - Generates a comprehensive data dictionary of your sage_intacct data through the [dbt docs site](https://fivetran.github.io/dbt_sage_intacct_source/). - These tables are designed to work simultaneously with our [Sage Intacct transformation package](https://github.com/fivetran/dbt_sage_intacct). @@ -39,7 +40,7 @@ If you are **not** using the [Sage Intacct transformation package](https://githu ```yaml packages: - package: fivetran/sage_intacct_source - version: [">=0.4.0", "<0.5.0"] # we recommend using ranges to capture non-breaking changes automatically + version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically ``` ### Step 3: Define database and schema variables By default, this package runs using your destination and the `sage_intacct` schema. If this is not where your Sage Intacct data is (for example, if your Sage Intacct schema is named `sage_intacct_fivetran`), add the following configuration to your root `dbt_project.yml` file: diff --git a/dbt_project.yml b/dbt_project.yml index 94764ce..40a41c1 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'sage_intacct_source' -version: '0.4.1' +version: '0.5.0' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index dfb0b91..5e98cd5 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,9 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True integration_tests: target: redshift diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f6ccdce..244eccc 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'sage_intacct_source_integration_tests' -version: '0.4.1' +version: '0.5.0' config-version: 2 profile: 'integration_tests' @@ -21,3 +21,6 @@ seeds: dispatch: - macro_namespace: dbt_utils search_order: ['spark_utils', 'dbt_utils'] + +flags: + send_anonymous_usage_stats: False diff --git a/models/src_sage_intacct.yml b/models/src_sage_intacct.yml index bc0a036..a401b17 100644 --- a/models/src_sage_intacct.yml +++ b/models/src_sage_intacct.yml @@ -6,9 +6,10 @@ sources: database: "{% if target.type != 'spark' %}{{var('sage_intacct_database', target.database)}}{% endif %}" loader: fivetran loaded_at_field: _fivetran_synced - freshness: - warn_after: {count: 48, period: hour} - error_after: {count: 72, period: hour} + config: + freshness: + warn_after: {count: 48, period: hour} + error_after: {count: 72, period: hour} tables: @@ -95,7 +96,8 @@ sources: - name: gl_account identifier: "{{ var('sage_intacct_gl_account_identifier', 'gl_account') }}" - freshness: null + config: + freshness: null description: Records each existing account, with their corresponding categories. columns: - name: recordno