updating to latest tuva project, and multi year pk fix

dbt_project.yml

@@ -5,14 +5,14 @@ config-version: 2
 profile: default                   
 
 vars:
-  input_database: medicare_lds_five_percent
-  input_schema:  raw_data
+  input_database: medicare_lds_five_multi_year
+  input_schema:  raw
 
   claims_enabled: true
 
   # Default values for data marts, update as needed
-  cms_hcc_payment_year: 2021
-  quality_measures_period_end: "2020-12-31"
+  cms_hcc_payment_year: 2024
+  quality_measures_period_end: "2023-12-31"
 
 model-paths: ["models"]
 analysis-paths: ["analyses"]
@@ -25,3 +25,8 @@ target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
   - "target"
   - "dbt_packages"
+
+models:
+  the_tuva_project:
+    data_quality:
+      +enabled: false

models/_models.yml

@@ -9,33 +9,49 @@ models:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%}
       materialized: table
+      tags: connector
+
 
   - name: medical_claim
     description: Claim line details of all claim types
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%}
       materialized: table
+      tags: connector      
+    columns:
+      - name: claim_id
+        description: Unique identifier for a claim.
+      - name: line_number
+        description: Line number associated with the claim.
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+           - claim_id
+           - claim_line_number      
 
   - name: pharmacy_claim
     description: Claim line details of all claim types
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%}
       materialized: table
+      tags: connector      
 
 ### intermediate
   - name: carrier_claim
     description: Claim line details of carrier claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: dme_claim
     description: Claim line details of durable medical equipment claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: eligibility_unpivot
     description: >
@@ -45,124 +61,145 @@ models:
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: home_health_claim
     description: Claim line details of home health claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: hospice_claim
     description: Claim line details of hospice claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: inpatient_claim
     description: Claim line details of inpatient claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: outpatient_claim
     description: Claim line details of outpatient claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
   - name: snf_claim
     description: Claim line details of skilled nursing facility claims
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_int_input_layer{% else %}_int_input_layer{%- endif -%}
+      tags: connector
 
 ### staging
   - name: stg_carrier_base_claim
     description: Staging carrier base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_carrier_claim_line
     description: Staging carrier claim line
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_dme_base_claim
     description: Staging dme base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_dme_claim_line
     description: Staging dme claim line
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_hha_base_claim
     description: Staging hha base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_hha_revenue_center
     description: Staging hha revenue center
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_hospice_base_claim
     description: Staging hospice base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_hospice_revenue_center
     description: Staging hospice revenue center
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_inpatient_base_claim
     description: Staging inpatient base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_inpatient_revenue_center
     description: Staging inpatient revenue center
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_master_beneficiary_summary
     description: Staging master beneficiary summary
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_outpatient_base_claim
     description: Staging outpatient base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_outpatient_revenue_center
     description: Staging outpatient revenue center
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_snf_base_claim
     description: Staging snf base claim
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector
 
   - name: stg_snf_revenue_center
     description: Staging snf revenue center
     config:
       schema: |
         {%- if var('tuva_schema_prefix',None) != None -%}_{{var('tuva_schema_prefix')}}_stg_input_layer{% else %}_stg_input_layer{%- endif -%}
+      tags: connector

models/final/eligibility.sql

@@ -223,4 +223,7 @@ select
     , data_source
     , file_name
     , ingest_datetime
+    , cast(null as date ) as file_date
+    , cast(null as {{ dbt.type_string() }}) as group_id
+    , cast(null as {{ dbt.type_string() }}) as group_name
 from joined

models/final/medical_claim.sql

@@ -61,7 +61,8 @@ select
     , c.discharge_disposition_code
     , c.place_of_service_code
     , c.bill_type_code
-    , c.ms_drg_code
+    , c.ms_drg_code as drg_code
+    , case when c.ms_drg_code is not null then 'ms-drg' else null end as drg_code_type
     , c.apr_drg_code
     , c.revenue_center_code
     , c.service_unit_quantity
@@ -190,6 +191,7 @@ select
     , c.data_source
     , c.file_name
     , c.ingest_datetime
+    , cast(null as date ) as file_date
 from unioned as c
     inner join member_months as mm
         on c.person_id = mm.person_id

models/final/pharmacy_claim.sql

@@ -22,5 +22,6 @@ select
     , cast(null as int) as in_network_flag
     , cast(null as {{ dbt.type_string() }}) as data_source
     , cast(null as {{ dbt.type_timestamp() }}) as file_name
+    , cast(null as date ) as file_date
     , cast(null as date ) as ingest_datetime
 limit 0

models/staging/stg_carrier_base_claim.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , clm_thru_dt
     , nch_near_line_rec_ident_cd
     , nch_clm_type_cd
@@ -46,7 +46,7 @@ select
     , icd_dgns_cd12
     , icd_dgns_vrsn_cd12
     , dob_dt
-    , gndr_cd
+    ----,gndr_cd
     , bene_race_cd
     , bene_cnty_cd
     , bene_state_cd
@@ -58,4 +58,5 @@ select
     , aco_id_num
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','carrier_base_claim') }}

models/staging/stg_carrier_claim_line.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , line_num
     , clm_thru_dt
     , nch_clm_type_cd
@@ -70,4 +70,5 @@ select
     , clm_next_gnrtn_aco_ind_cd5
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','carrier_claim_line') }}

models/staging/stg_dme_base_claim.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , clm_thru_dt
     , nch_near_line_rec_ident_cd
     , nch_clm_type_cd
@@ -45,7 +45,7 @@ select
     , rfr_physn_upin
     , rfr_physn_npi
     , dob_dt
-    , gndr_cd
+    --,gndr_cd
     , bene_race_cd
     , bene_cnty_cd
     , bene_state_cd
@@ -54,4 +54,5 @@ select
     , aco_id_num
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','dme_base_claim') }}

models/staging/stg_dme_claim_line.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , clm_line_num
     , clm_thru_dt
     , nch_clm_type_cd
@@ -58,4 +58,5 @@ select
     , line_othr_apld_amt7
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','dme_claim_line') }}

models/staging/stg_hha_base_claim.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , prvdr_num
     , clm_thru_dt
     , nch_near_line_rec_ident_cd
@@ -74,7 +74,7 @@ select
     , clm_hha_tot_visit_cnt
     , clm_admsn_dt
     , dob_dt
-    , gndr_cd
+    --,gndr_cd
     , bene_race_cd
     , bene_cnty_cd
     , bene_state_cd
@@ -93,4 +93,5 @@ select
     , aco_id_num
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','hha_base_claim') }}

models/staging/stg_hha_revenue_center.sql

@@ -1,6 +1,6 @@
 select
       desy_sort_key
-    , claim_no
+    , concat(claim_no,'|',regexp_substr(file_name, '20[0-9]{2}')) as claim_no
     , clm_line_num
     , clm_thru_dt
     , nch_clm_type_cd
@@ -31,4 +31,5 @@ select
     , thrpy_cap_ind_cd2
     , file_name
     , ingest_datetime
+    , regexp_substr(file_name, '20[0-9]{2}') AS year_nbr
 from {{ source('medicare_lds','hha_revenue_center') }}