Skip to content

Commit 9ecd6d0

Browse files
tkirschketkirschke
authored
Merge pull request #274 from ScalefreeCOM/140-Optionality-for-NHLink-foreign-hashkeys
140 optionality for nh link foreign hashkeys
2 parents a80a51b + 4fae523 commit 9ecd6d0

File tree

10 files changed

+119
-64
lines changed

10 files changed

+119
-64
lines changed

‎macros/tables/bigquery/nh_link.sql

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,10 @@
11
{%- macro default__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
32

4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
93
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
104

115
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
126
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
137

14-
158
{# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
169
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
1710
{# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -21,10 +14,21 @@
2114

2215
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2316
{%- set source_models = source_model_values['source_model_list'] -%}
17+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
18+
{%- for source_model in source_models -%}
19+
{%- set source_relation = ref(source_model.name) -%}
20+
{%- endfor -%}
21+
22+
{%- if execute -%}
23+
2424
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2525
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2626
{{ log('source_models: '~source_models, false) }}
2727

28+
29+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
30+
{%- set foreign_hashkeys = [] -%}
31+
{%- endif -%}
2832
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2933

3034
{{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +255,6 @@ records_to_insert AS (
251255

252256
SELECT * FROM records_to_insert
253257

258+
{%- endif -%}
259+
254260
{%- endmacro -%}

‎macros/tables/databricks/nh_link.sql

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
{%- macro databricks__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
32

4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
93
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
104

115
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
126
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
13-
7+
{{ log('source_models: '~source_models, false) }}
148

159
{# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
1610
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
@@ -21,10 +15,20 @@
2115

2216
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2317
{%- set source_models = source_model_values['source_model_list'] -%}
18+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
19+
{%- for source_model in source_models -%}
20+
{%- set source_relation = ref(source_model.name) -%}
21+
{%- endfor -%}
22+
23+
{%- if execute -%}
24+
2425
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2526
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2627
{{ log('source_models: '~source_models, false) }}
2728

29+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
30+
{%- set foreign_hashkeys = [] -%}
31+
{%- endif -%}
2832
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2933

3034
{{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +255,6 @@ records_to_insert AS (
251255

252256
SELECT * FROM records_to_insert
253257

258+
{%- endif -%}
259+
254260
{%- endmacro -%}

‎macros/tables/exasol/nh_link.sql

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
66
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
77

8-
98
{# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
109
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
1110
{# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -15,10 +14,21 @@
1514

1615
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
1716
{%- set source_models = source_model_values['source_model_list'] -%}
17+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
18+
{%- for source_model in source_models -%}
19+
{%- set source_relation = ref(source_model.name) -%}
20+
{%- endfor -%}
21+
22+
{%- if execute -%}
23+
1824
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
1925
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2026
{{ log('source_models: '~source_models, false) }}
2127

28+
29+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
30+
{%- set foreign_hashkeys = [] -%}
31+
{%- endif -%}
2232
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2333

2434
{{ datavault4dbt.prepend_generated_by() }}
@@ -245,4 +255,6 @@ records_to_insert AS (
245255

246256
SELECT * FROM records_to_insert
247257

258+
{% endif %}
259+
248260
{%- endmacro -%}

‎macros/tables/fabric/nh_link.sql

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
11
{%- macro fabric__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
22

3-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
4-
5-
{%- if execute -%}
6-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
7-
{%- endif %}
8-
9-
{%- endif -%}
10-
113
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
124

135
{%- set beginning_of_all_times = datavault4dbt.beginning_of_all_times() -%}
@@ -24,10 +16,20 @@
2416

2517
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2618
{%- set source_models = source_model_values['source_model_list'] -%}
19+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
20+
{%- for source_model in source_models -%}
21+
{%- set source_relation = ref(source_model.name) -%}
22+
{%- endfor -%}
23+
24+
{%- if execute -%}
25+
2726
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2827
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2928
{{ log('source_models: '~source_models, false) }}
3029

30+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
31+
{%- set foreign_hashkeys = [] -%}
32+
{%- endif -%}
3133
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
3234

3335
{%- set final_columns_to_select = datavault4dbt.escape_column_names(final_columns_to_select) -%}
@@ -273,5 +275,6 @@ records_to_insert AS (
273275
)
274276

275277
SELECT * FROM records_to_insert
278+
{% endif %}
276279

277280
{%- endmacro -%}

‎macros/tables/nh_link.sql

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,7 @@
8282

8383
#}
8484

85-
{%- macro nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts=none, src_rsrc=none, disable_hwm=false, source_is_single_batch=false) -%}
86-
85+
{%- macro nh_link(link_hashkey, payload, source_models, foreign_hashkeys=none, src_ldts=none, src_rsrc=none, disable_hwm=false, source_is_single_batch=false) -%}
8786
{# Applying the default aliases as stored inside the global variables, if src_ldts and src_rsrc are not set. #}
8887

8988
{%- set src_ldts = datavault4dbt.replace_standard(src_ldts, 'datavault4dbt.ldts_alias', 'ldts') -%}

‎macros/tables/oracle/nh_link.sql

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
{%- macro oracle__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
3-
4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
92

103
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
114

125
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
136
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
7+
{{ log('source_models: '~source_models, false) }}
148

159
{# If no specific link_hk and fk_columns are defined for each source, we apply the values set in the link_hashkey and foreign_hashkeys variable. #}
1610
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
@@ -21,10 +15,21 @@
2115

2216
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2317
{%- set source_models = source_model_values['source_model_list'] -%}
18+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
19+
{%- for source_model in source_models -%}
20+
{%- set source_relation = ref(source_model.name) -%}
21+
{%- endfor -%}
22+
23+
{%- if execute -%}
24+
2425
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2526
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2627
{{ log('source_models: '~source_models, false) }}
2728

29+
30+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
31+
{%- set foreign_hashkeys = [] -%}
32+
{%- endif -%}
2833
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2934

3035
{{ datavault4dbt.prepend_generated_by() }}
@@ -257,4 +262,6 @@ records_to_insert AS (
257262

258263
SELECT * FROM records_to_insert
259264

265+
{%- endif -%}
266+
260267
{%- endmacro -%}

‎macros/tables/postgres/nh_link.sql

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
11
{%- macro postgres__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
32

4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
93
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
104

115
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
126
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
13-
7+
{{ log('source_models: '~source_models, false) }}
148

159
{# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
1610
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
@@ -21,10 +15,21 @@
2115

2216
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2317
{%- set source_models = source_model_values['source_model_list'] -%}
18+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
19+
{%- for source_model in source_models -%}
20+
{%- set source_relation = ref(source_model.name) -%}
21+
{%- endfor -%}
22+
23+
{%- if execute -%}
24+
2425
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2526
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2627
{{ log('source_models: '~source_models, false) }}
2728

29+
30+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
31+
{%- set foreign_hashkeys = [] -%}
32+
{%- endif -%}
2833
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2934

3035
{{ datavault4dbt.prepend_generated_by() }}
@@ -255,4 +260,6 @@ records_to_insert AS (
255260

256261
SELECT * FROM records_to_insert
257262

263+
{%- endif -%}
264+
258265
{%- endmacro -%}

‎macros/tables/redshift/nh_link.sql

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,10 @@
11
{%- macro redshift__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
32

4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
93
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
104

115
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
126
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
137

14-
158
{# If no specific link_hk, fk_columns, or payload are defined for each source, we apply the values set in the link_hashkey, foreign_hashkeys, and payload variable. #}
169
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
1710
{# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
@@ -21,10 +14,21 @@
2114

2215
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2316
{%- set source_models = source_model_values['source_model_list'] -%}
17+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
18+
{%- for source_model in source_models -%}
19+
{%- set source_relation = ref(source_model.name) -%}
20+
{%- endfor -%}
21+
22+
{%- if execute -%}
23+
2424
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2525
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2626
{{ log('source_models: '~source_models, false) }}
2727

28+
29+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
30+
{%- set foreign_hashkeys = [] -%}
31+
{%- endif -%}
2832
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2933

3034
{{ datavault4dbt.prepend_generated_by() }}
@@ -255,4 +259,6 @@ records_to_insert AS (
255259

256260
SELECT * FROM records_to_insert
257261

262+
{% endif %}
263+
258264
{%- endmacro -%}

‎macros/tables/snowflake/nh_link.sql

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,33 @@
11
{%- macro snowflake__nh_link(link_hashkey, foreign_hashkeys, payload, source_models, src_ldts, src_rsrc, disable_hwm, source_is_single_batch) -%}
2-
{%- if not (foreign_hashkeys is iterable and foreign_hashkeys is not string) -%}
3-
4-
{%- if execute -%}
5-
{{ exceptions.raise_compiler_error("Only one foreign key provided for this link. At least two required.") }}
6-
{%- endif %}
7-
8-
{%- endif -%}
92

103
{%- set ns = namespace(last_cte= "", source_included_before = {}, has_rsrc_static_defined=true, source_models_rsrc_dict={}) -%}
114

125
{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%}
136
{%- set timestamp_format = datavault4dbt.timestamp_format() -%}
7+
{{ log('source_models: '~source_models, false) }}
148

159
{# If no specific link_hk and fk_columns are defined for each source, we apply the values set in the link_hashkey and foreign_hashkeys variable. #}
1610
{# If no rsrc_static parameter is defined in ANY of the source models then the whole code block of record_source performance lookup is not executed #}
1711
{# For the use of record_source performance lookup it is required that every source model has the parameter rsrc_static defined and it cannot be an empty string #}
1812
{%- if source_models is not mapping and not datavault4dbt.is_list(source_models) -%}
1913
{%- set source_models = {source_models: {}} -%}
2014
{%- endif -%}
21-
2215
{%- set source_model_values = fromjson(datavault4dbt.source_model_processing(source_models=source_models, parameters={'link_hk':link_hashkey}, foreign_hashkeys=foreign_hashkeys, payload=payload)) -%}
2316
{%- set source_models = source_model_values['source_model_list'] -%}
17+
{#This loop goes through each source_model in the source_models list. For each model, it uses the ref() function to establish dependencies for dbt to track the relationships between models..#}
18+
{%- for source_model in source_models -%}
19+
{%- set source_relation = ref(source_model.name) -%}
20+
{%- endfor -%}
21+
22+
{%- if execute -%}
23+
2424
{%- set ns.has_rsrc_static_defined = source_model_values['has_rsrc_static_defined'] -%}
2525
{%- set ns.source_models_rsrc_dict = source_model_values['source_models_rsrc_dict'] -%}
2626
{{ log('source_models: '~source_models, false) }}
2727

28+
{%- if not datavault4dbt.is_something(foreign_hashkeys) -%}
29+
{%- set foreign_hashkeys = [] -%}
30+
{%- endif -%}
2831
{%- set final_columns_to_select = [link_hashkey] + foreign_hashkeys + [src_ldts] + [src_rsrc] + payload -%}
2932

3033
{{ datavault4dbt.prepend_generated_by() }}
@@ -251,4 +254,6 @@ records_to_insert AS (
251254

252255
SELECT * FROM records_to_insert
253256

257+
{%- endif -%}
258+
254259
{%- endmacro -%}

0 commit comments

Comments
 (0)