Skip to content

Commit ca5d7ac

Browse files
tkirschketkirschke
authored
Merge pull request #251 from ScalefreeCOM/add-ghost-record-numeric-values-variables
Change ghost record numeric defaults and add ghost record numeric values variables
2 parents 846a926 + b40a8c8 commit ca5d7ac

File tree

2 files changed

+49
-26
lines changed

2 files changed

+49
-26
lines changed

‎dbt_project.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ vars:
5858
datavault4dbt.error_value_alt__STRING: 'e'
5959
datavault4dbt.unknown_value__STRING: '(unknown)'
6060
datavault4dbt.unknown_value_alt__STRING: 'u'
61+
datavault4dbt.unknown_value__numeric: '-1'
62+
datavault4dbt.error_value__numeric: '-2'
6163

6264

6365
# Configuring models

‎macros/supporting/ghost_record_per_datatype.sql

Lines changed: 47 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,27 @@
2424

2525
{%- set datatype = datatype | string | upper | trim -%}
2626

27+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
28+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}
29+
2730
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
2831
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
32+
2933
{%- if ghost_record_type == 'unknown' -%}
3034
{%- if datatype == 'TIMESTAMP' %} {{ datavault4dbt.string_to_timestamp( timestamp_format , beginning_of_all_times) }} as {{ alias }}
3135
{%- elif datatype == 'DATE'-%} PARSE_DATE('{{date_format}}','{{ beginning_of_all_times_date }}') as {{ alias }}
3236
{%- elif datatype == 'STRING' %} '{{unknown_value__STRING}}' as {{ alias }}
33-
{%- elif datatype == 'INT64' %} CAST('0' as INT64) as {{ alias }}
34-
{%- elif datatype == 'FLOAT64' %} CAST('0' as FLOAT64) as {{ alias }}
37+
{%- elif datatype == 'INT64' %} CAST({{unknown_value__numeric}} as INT64) as {{ alias }}
38+
{%- elif datatype == 'FLOAT64' %} CAST({{unknown_value__numeric}} as FLOAT64) as {{ alias }}
3539
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
3640
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
3741
{% endif %}
3842
{%- elif ghost_record_type == 'error' -%}
3943
{%- if datatype == 'TIMESTAMP' %} {{ datavault4dbt.string_to_timestamp( timestamp_format , end_of_all_times) }} as {{ alias }}
4044
{%- elif datatype == 'DATE'-%} PARSE_DATE('{{date_format}}', '{{ end_of_all_times_date }}') as {{ alias }}
4145
{%- elif datatype == 'STRING' %} '{{error_value__STRING}}' as {{ alias }}
42-
{%- elif datatype == 'INT64' %} CAST('-1' as INT64) as {{ alias }}
43-
{%- elif datatype == 'FLOAT64' %} CAST('-1' as FLOAT64) as {{ alias }}
46+
{%- elif datatype == 'INT64' %} CAST({{error_value__numeric}} as INT64) as {{ alias }}
47+
{%- elif datatype == 'FLOAT64' %} CAST({{error_value__numeric}} as FLOAT64) as {{ alias }}
4448
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
4549
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
4650
{% endif %}
@@ -62,6 +66,9 @@
6266
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
6367
{%- set date_format = datavault4dbt.date_format() -%}
6468

69+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
70+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}
71+
6572
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
6673
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
6774
{%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%}
@@ -94,8 +101,8 @@
94101
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as {{ alias }}
95102
{%- endif -%}
96103
{%- elif datatype.upper().startswith('CHAR') -%} CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
97-
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('0' as {{ datatype }}) as {{ alias }}
98-
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('0' as DOUBLE PRECISION) as {{ alias }}
104+
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('{{unknown_value__numeric}}' as {{ datatype }}) as {{ alias }}
105+
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('{{unknown_value__numeric}}' as DOUBLE PRECISION) as {{ alias }}
99106
{%- elif datatype == 'BOOLEAN' %} FALSE as {{ alias }}
100107
{%- elif datatype.upper().startswith('HASHTYPE') -%} CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
101108
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
@@ -122,8 +129,8 @@
122129
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as {{ alias }}
123130
{%- endif -%}
124131
{%- elif datatype.upper().startswith('CHAR') -%} CAST('{{ error_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
125-
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('-1' as {{ datatype }}) as {{ alias }}
126-
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('-1' as DOUBLE PRECISION) as {{ alias }}
132+
{%- elif datatype.upper().startswith('DECIMAL') -%} CAST('{{error_value__numeric}}' as {{ datatype }}) as {{ alias }}
133+
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('{{error_value__numeric}}' as DOUBLE PRECISION) as {{ alias }}
127134
{%- elif datatype == 'BOOLEAN' %} FALSE as {{ alias }}
128135
{%- elif datatype.upper().startswith('HASHTYPE') -%} CAST('{{ error_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
129136
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
@@ -151,6 +158,9 @@
151158
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
152159
{%- set date_format = datavault4dbt.date_format() -%}
153160

161+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
162+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}
163+
154164
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
155165
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
156166
{%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%}
@@ -180,7 +190,7 @@
180190
{%- else -%}
181191
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as {{ alias }}
182192
{%- endif -%}
183-
{%- elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}0 AS {{ alias }}
193+
{%- elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}{{unknown_value__numeric}} AS {{ alias }}
184194
{%- elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }}
185195
{%- else %}NULL AS {{ alias }}
186196
{% endif %}
@@ -205,7 +215,7 @@
205215
{%- else -%}
206216
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as {{ alias }}
207217
{%- endif -%}
208-
{% elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}-1 AS {{ alias }}
218+
{% elif datatype in ['NUMBER','INT','FLOAT','DECIMAL'] %}{{error_value__numeric}} AS {{ alias }}
209219
{% elif datatype == 'BOOLEAN' %}CAST('FALSE' AS BOOLEAN) AS {{ alias }}
210220
{% else %}NULL AS {{ alias }}
211221
{% endif %}
@@ -226,7 +236,9 @@
226236

227237
{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%}
228238
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
229-
{%- set date_format = datavault4dbt.date_format() -%}
239+
240+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
241+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}
230242

231243
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
232244
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
@@ -263,7 +275,8 @@
263275
CAST('{{ unknown_value__STRING }}' as {{ datatype }} ) as "{{ alias }}"
264276
{%- endif -%}
265277
{%- elif datatype == 'TINYINT' -%} CAST('254' as {{ datatype }}) as "{{ alias }}"
266-
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype or datatype == 'BIT'%} CAST('-1' as {{ datatype }}) as "{{ alias }}"
278+
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('{{unknown_value__numeric}}' as {{ datatype }}) as "{{ alias }}"
279+
{%- elif datatype == 'BIT' -%} CAST(0 as {{ datatype }}) as "{{ alias }}"
267280
{%- elif datatype == 'DATE'-%} CONVERT(DATE, '{{ beginning_of_all_times_date }}') as "{{ alias }}"
268281
{%- elif 'BINARY' in datatype -%}
269282
CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as "{{ alias }}"
@@ -292,7 +305,7 @@
292305
CAST('{{ error_value__STRING }}' as {{ datatype }} ) as "{{ alias }}"
293306
{%- endif -%}
294307
{%- elif datatype == 'TINYINT' -%} CAST('255' as {{ datatype }}) as "{{ alias }}"
295-
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('-2' as {{ datatype }}) as "{{ alias }}"
308+
{%- elif 'INT' in datatype or datatype == 'DECIMAL' or datatype == 'NUMERIC' or 'MONEY' in datatype %} CAST('{{error_value__numeric}}' as {{ datatype }}) as "{{ alias }}"
296309
{%- elif datatype == 'BIT' -%} CAST(0 as {{ datatype }}) as "{{ alias }}"
297310
{%- elif datatype == 'DATE'-%} CONVERT(DATE, '{{ end_of_all_times_date }}') as "{{ alias }}"
298311
{%- elif 'BINARY' in datatype -%}
@@ -320,27 +333,32 @@
320333
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
321334
{%- set date_format = datavault4dbt.date_format() -%}
322335

336+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', '-1') -%}
337+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', '-2') -%}
338+
323339
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
324340
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
325341
{%- set datatype = datatype | string | upper | trim -%}
326342

327343
{%- if ghost_record_type == 'unknown' -%}
328344
{%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, beginning_of_all_times_date) }} AS {{ alias }}
345+
{%- elif datatype == 'TIME WITH TIME ZONE' %} CAST('00:00:01 UTC' as TIMETZ) as {{ alias }}
346+
{%- elif datatype == 'TIME WITHOUT TIME ZONE' %} CAST('00:00:01' as TIME) as {{ alias }}
329347
{%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
330-
{%- elif datatype == 'TEXT' %} CAST('{{unknown_value__STRING}}' as TEXT) as {{ alias }}
331-
{%- elif datatype == 'VARCHAR' %} CAST('{{unknown_value__STRING}}' as VARCHAR) as {{ alias }}
332-
{%- elif datatype == 'INTEGER' %} CAST('0' as INTEGER) as {{ alias }}
333-
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('0' as DOUBLE PRECISION) as {{ alias }}
348+
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{unknown_value__STRING}}' as {{ alias }}
349+
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
350+
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
334351
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
335352
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
336353
{% endif %}
337-
{%- elif ghost_record_type == 'error' -%}
354+
{%- elif ghost_record_type == 'error' -%}
338355
{%- if 'TIMESTAMP' in datatype %}{{ datavault4dbt.string_to_timestamp(timestamp_format, end_of_all_times) }} as {{ alias }}
356+
{%- elif datatype == 'TIME WITH TIME ZONE' %} CAST('23:59:59 UTC' as TIMETZ) as {{ alias }}
357+
{%- elif datatype == 'TIME WITHOUT TIME ZONE' %} CAST('23:59:59' as TIME) as {{ alias }}
339358
{%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
340-
{%- elif datatype == 'TEXT' %} CAST('{{error_value__STRING}}' as TEXT) as {{ alias }}
341-
{%- elif datatype == 'VARCHAR' %} CAST('{{error_value__STRING}}' as VARCHAR) as {{ alias }}
342-
{%- elif datatype == 'INTEGER' %} CAST('-1' as INTEGER) as {{ alias }}
343-
{%- elif datatype == 'DOUBLE PRECISION' %} CAST('-1' as DOUBLE PRECISION) as {{ alias }}
359+
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{error_value__STRING}}' as {{ alias }}
360+
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
361+
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
344362
{%- elif datatype == 'BOOLEAN' %} CAST('FALSE' as BOOLEAN) as {{ alias }}
345363
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
346364
{% endif %}
@@ -362,6 +380,9 @@
362380
{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%}
363381
{%- set date_format = datavault4dbt.date_format() -%}
364382

383+
{%- set unknown_value__numeric = var('datavault4dbt.unknown_value__numeric', -1) -%}
384+
{%- set error_value__numeric = var('datavault4dbt.error_value__numeric', -2) -%}
385+
365386
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
366387
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
367388

@@ -379,8 +400,8 @@
379400
{%- elif datatype == 'TIME' %} CAST('00:00:01' as TIME) as {{ alias }}
380401
{%- elif datatype == 'DATE'-%} TO_DATE('{{ beginning_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
381402
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{unknown_value__STRING}}' as {{ alias }}
382-
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(0 as {{ datatype }}) as {{ alias }}
383-
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(0 as {{ datatype }}) as {{ alias }}
403+
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
404+
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{unknown_value__numeric}} as {{ datatype }}) as {{ alias }}
384405
{%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('FALSE' as BOOLEAN) as {{ alias }}
385406
{%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ unknown_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
386407
{%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }}
@@ -392,8 +413,8 @@
392413
{%- elif datatype == 'TIME' %} CAST('23:59:59' as TIME) as {{ alias }}
393414
{%- elif datatype == 'DATE'-%} TO_DATE('{{ end_of_all_times_date }}', '{{ date_format }}' ) as {{ alias }}
394415
{%- elif 'CHAR' in datatype or datatype == 'TEXT' %} '{{error_value__STRING}}' as {{ alias }}
395-
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST(-1 as {{ datatype }}) as {{ alias }}
396-
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST(-1 as {{ datatype }}) as {{ alias }}
416+
{%- elif datatype in ['INTEGER', 'INT', 'INT2', 'INT4', 'INT8', 'SMALLINT', 'BIGINT', 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'DOUBLE', 'FLOAT', 'FLOAT8'] %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
417+
{%- elif 'DECIMAL' in datatype or 'NUMERIC' in datatype %} CAST({{error_value__numeric}} as {{ datatype }}) as {{ alias }}
397418
{%- elif datatype in ['BOOLEAN', 'BOOL'] %} CAST('FALSE' as BOOLEAN) as {{ alias }}
398419
{%- elif datatype in ['VARBYTE', 'VARBINARY', 'BINARY VARYING'] %} CAST('{{ error_value__HASHTYPE }}' as {{ datatype }}) as {{ alias }}
399420
{%- elif datatype == 'GEOMETRY' %} CAST(ST_POINT(0, 90) as {{ datatype }}) as {{ alias }}

0 commit comments

Comments
 (0)