diff --git a/dbt_project.yml b/dbt_project.yml index da69128..272f76b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,14 +5,14 @@ config-version: 2 profile: default vars: - input_database: medicare_lds_five_percent - input_schema: raw_data + input_database: medicare_lds_five_multi_year + input_schema: raw claims_enabled: true # Default values for data marts, update as needed - cms_hcc_payment_year: 2021 - quality_measures_period_end: "2020-12-31" + cms_hcc_payment_year: 2024 + quality_measures_period_end: "2023-12-31" model-paths: ["models"] analysis-paths: ["analyses"] @@ -25,3 +25,8 @@ target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" + +models: + the_tuva_project: + data_quality: + +enabled: false \ No newline at end of file diff --git a/models/_models.yml b/models/_models.yml index e6b9a5c..abb1498 100644 --- a/models/_models.yml +++ b/models/_models.yml @@ -9,6 +9,8 @@ models: schema: | {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%} materialized: table + tags: connector + - name: medical_claim description: Claim line details of all claim types @@ -16,6 +18,17 @@ models: schema: | {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%} materialized: table + tags: connector + columns: + - name: claim_id + description: Unique identifier for a claim. + - name: line_number + description: Line number associated with the claim. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - claim_id + - claim_line_number - name: pharmacy_claim description: Claim line details of all claim types @@ -23,6 +36,7 @@ models: schema: | {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%} materialized: table + tags: connector ### intermediate - name: carrier_claim @@ -30,12 +44,14 @@ models: config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: dme_claim description: Claim line details of durable medical equipment claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: eligibility_unpivot description: > @@ -45,36 +61,42 @@ models: config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: home_health_claim description: Claim line details of home health claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: hospice_claim description: Claim line details of hospice claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: inpatient_claim description: Claim line details of inpatient claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: outpatient_claim description: Claim line details of outpatient claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector - name: snf_claim description: Claim line details of skilled nursing facility claims config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%} + tags: connector ### staging - name: stg_carrier_base_claim @@ -82,87 +104,102 @@ models: config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_carrier_claim_line description: Staging carrier claim line config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_dme_base_claim description: Staging dme base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_dme_claim_line description: Staging dme claim line config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_hha_base_claim description: Staging hha base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_hha_revenue_center description: Staging hha revenue center config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_hospice_base_claim description: Staging hospice base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_hospice_revenue_center description: Staging hospice revenue center config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_inpatient_base_claim description: Staging inpatient base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_inpatient_revenue_center description: Staging inpatient revenue center config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_master_beneficiary_summary description: Staging master beneficiary summary config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_outpatient_base_claim description: Staging outpatient base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_outpatient_revenue_center description: Staging outpatient revenue center config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_snf_base_claim description: Staging snf base claim config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector - name: stg_snf_revenue_center description: Staging snf revenue center config: schema: | {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%} + tags: connector \ No newline at end of file diff --git a/models/final/eligibility.sql b/models/final/eligibility.sql index f4f0492..5f590d0 100644 --- a/models/final/eligibility.sql +++ b/models/final/eligibility.sql @@ -223,4 +223,7 @@ select , data_source , file_name , ingest_datetime + , cast(null as date ) as file_date + , cast(null as {{ dbt.type_string() }}) as group_id + , cast(null as {{ dbt.type_string() }}) as group_name from joined \ No newline at end of file diff --git a/models/final/medical_claim.sql b/models/final/medical_claim.sql index 3e21fb4..fbd7f8f 100644 --- a/models/final/medical_claim.sql +++ b/models/final/medical_claim.sql @@ -61,7 +61,8 @@ select , c.discharge_disposition_code , c.place_of_service_code , c.bill_type_code - , c.ms_drg_code + , c.ms_drg_code as drg_code + , case when c.ms_drg_code is not null then 'ms-drg' else null end as drg_code_type , c.apr_drg_code , c.revenue_center_code , c.service_unit_quantity @@ -190,6 +191,7 @@ select , c.data_source , c.file_name , c.ingest_datetime + , cast(null as date ) as file_date from unioned as c inner join member_months as mm on c.person_id = mm.person_id diff --git a/models/final/pharmacy_claim.sql b/models/final/pharmacy_claim.sql index 3d355a1..e5a0ac1 100644 --- a/models/final/pharmacy_claim.sql +++ b/models/final/pharmacy_claim.sql @@ -22,5 +22,6 @@ select , cast(null as int) as in_network_flag , cast(null as {{ dbt.type_string() }}) as data_source , cast(null as {{ dbt.type_timestamp() }}) as file_name + , cast(null as date ) as file_date , cast(null as date ) as ingest_datetime limit 0 \ No newline at end of file diff --git a/models/staging/stg_carrier_base_claim.sql b/models/staging/stg_carrier_base_claim.sql index e63dc04..09e1b95 100644 --- a/models/staging/stg_carrier_base_claim.sql +++ b/models/staging/stg_carrier_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_thru_dt , nch_near_line_rec_ident_cd , nch_clm_type_cd @@ -46,7 +46,7 @@ select , icd_dgns_cd12 , icd_dgns_vrsn_cd12 , dob_dt - , gndr_cd + ----,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -58,4 +58,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','carrier_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_carrier_claim_line.sql b/models/staging/stg_carrier_claim_line.sql index aeb374c..d0a0aed 100644 --- a/models/staging/stg_carrier_claim_line.sql +++ b/models/staging/stg_carrier_claim_line.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , line_num , clm_thru_dt , nch_clm_type_cd @@ -70,4 +70,5 @@ select , clm_next_gnrtn_aco_ind_cd5 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','carrier_claim_line') }} \ No newline at end of file diff --git a/models/staging/stg_dme_base_claim.sql b/models/staging/stg_dme_base_claim.sql index cbd40c1..adc5f59 100644 --- a/models/staging/stg_dme_base_claim.sql +++ b/models/staging/stg_dme_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_thru_dt , nch_near_line_rec_ident_cd , nch_clm_type_cd @@ -45,7 +45,7 @@ select , rfr_physn_upin , rfr_physn_npi , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -54,4 +54,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','dme_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_dme_claim_line.sql b/models/staging/stg_dme_claim_line.sql index e5af3af..697d233 100644 --- a/models/staging/stg_dme_claim_line.sql +++ b/models/staging/stg_dme_claim_line.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -58,4 +58,5 @@ select , line_othr_apld_amt7 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','dme_claim_line') }} \ No newline at end of file diff --git a/models/staging/stg_hha_base_claim.sql b/models/staging/stg_hha_base_claim.sql index 0604886..48bae1e 100644 --- a/models/staging/stg_hha_base_claim.sql +++ b/models/staging/stg_hha_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , prvdr_num , clm_thru_dt , nch_near_line_rec_ident_cd @@ -74,7 +74,7 @@ select , clm_hha_tot_visit_cnt , clm_admsn_dt , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -93,4 +93,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','hha_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_hha_revenue_center.sql b/models/staging/stg_hha_revenue_center.sql index 776923e..033554b 100644 --- a/models/staging/stg_hha_revenue_center.sql +++ b/models/staging/stg_hha_revenue_center.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -31,4 +31,5 @@ select , thrpy_cap_ind_cd2 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','hha_revenue_center') }} \ No newline at end of file diff --git a/models/staging/stg_hospice_base_claim.sql b/models/staging/stg_hospice_base_claim.sql index adda266..131be58 100644 --- a/models/staging/stg_hospice_base_claim.sql +++ b/models/staging/stg_hospice_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , prvdr_num , clm_thru_dt , nch_near_line_rec_ident_cd @@ -74,7 +74,7 @@ select , clm_hospc_start_dt_id , bene_hospc_prd_cnt , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -91,4 +91,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','hospice_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_hospice_revenue_center.sql b/models/staging/stg_hospice_revenue_center.sql index 07742e6..b45de15 100644 --- a/models/staging/stg_hospice_revenue_center.sql +++ b/models/staging/stg_hospice_revenue_center.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -28,4 +28,5 @@ select , thrpy_cap_ind_cd2 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','hospice_revenue_center') }} \ No newline at end of file diff --git a/models/staging/stg_inpatient_base_claim.sql b/models/staging/stg_inpatient_base_claim.sql index 2a1bd2e..9ca7e50 100644 --- a/models/staging/stg_inpatient_base_claim.sql +++ b/models/staging/stg_inpatient_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , prvdr_num , clm_thru_dt , nch_near_line_rec_ident_cd @@ -190,7 +190,7 @@ select , icd_prcdr_cd25 , prcdr_dt25 , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -230,4 +230,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','inpatient_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_inpatient_revenue_center.sql b/models/staging/stg_inpatient_revenue_center.sql index ec2493d..343a240 100644 --- a/models/staging/stg_inpatient_revenue_center.sql +++ b/models/staging/stg_inpatient_revenue_center.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -24,4 +24,5 @@ select , thrpy_cap_ind_cd2 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','inpatient_revenue_center') }} \ No newline at end of file diff --git a/models/staging/stg_master_beneficiary_summary.sql b/models/staging/stg_master_beneficiary_summary.sql index a2c443c..66c4a85 100644 --- a/models/staging/stg_master_beneficiary_summary.sql +++ b/models/staging/stg_master_beneficiary_summary.sql @@ -80,4 +80,5 @@ select , dual_stus_cd_12 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','master_beneficiary_summary') }} \ No newline at end of file diff --git a/models/staging/stg_outpatient_base_claim.sql b/models/staging/stg_outpatient_base_claim.sql index f29dbe1..f876d7b 100644 --- a/models/staging/stg_outpatient_base_claim.sql +++ b/models/staging/stg_outpatient_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , prvdr_num , clm_thru_dt , nch_near_line_rec_ident_cd @@ -132,7 +132,7 @@ select , clm_op_prvdr_pmt_amt , clm_op_bene_pmt_amt , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -151,4 +151,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','outpatient_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_outpatient_revenue_center.sql b/models/staging/stg_outpatient_revenue_center.sql index 88171da..4ebd2a3 100644 --- a/models/staging/stg_outpatient_revenue_center.sql +++ b/models/staging/stg_outpatient_revenue_center.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -42,4 +42,5 @@ select , rc_ptnt_add_on_pymt_amt , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','outpatient_revenue_center') }} \ No newline at end of file diff --git a/models/staging/stg_snf_base_claim.sql b/models/staging/stg_snf_base_claim.sql index 1b1bfdc..a9e3672 100644 --- a/models/staging/stg_snf_base_claim.sql +++ b/models/staging/stg_snf_base_claim.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , prvdr_num , clm_thru_dt , nch_near_line_rec_ident_cd @@ -147,7 +147,7 @@ select , icd_prcdr_cd25 , prcdr_dt25 , dob_dt - , gndr_cd + --,gndr_cd , bene_race_cd , bene_cnty_cd , bene_state_cd @@ -163,4 +163,5 @@ select , aco_id_num , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','snf_base_claim') }} \ No newline at end of file diff --git a/models/staging/stg_snf_revenue_center.sql b/models/staging/stg_snf_revenue_center.sql index c5863e6..e056ae1 100644 --- a/models/staging/stg_snf_revenue_center.sql +++ b/models/staging/stg_snf_revenue_center.sql @@ -1,6 +1,6 @@ select desy_sort_key - , claim_no + , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no , clm_line_num , clm_thru_dt , nch_clm_type_cd @@ -23,4 +23,5 @@ select , thrpy_cap_ind_cd2 , file_name , ingest_datetime + , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr from {{ source('medicare_lds','snf_revenue_center') }} \ No newline at end of file diff --git a/packages.yml b/packages.yml index db557d8..6e68928 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.9.2","<1.3.0"] - - package: tuva-health/the_tuva_project - version: [">=0.13.0","<0.14.0"] + - git: "https://github.com/tuva-health/tuva.git" + revision: benchmarks # this is up to date with v0.14.11