Fix enrollment being exploded to row per member-month. Adjust testing on PK. (#885)

* Fix enrollment being exploded to row per member-month. Adjusted unique value check on enrollment and member_month.

* Linting

* Revert testing changes

* Added member_months to models.yml

* Add stg core member months to models yml

Author: David Shimamoto

.gitignore

@@ -15,6 +15,7 @@ release_notes/
 # IDE and editor files
 .idea/
 .vscode/
+dbt_internal_packages/
 
 # Operating system files
 .DS_Store

macros/cross_database_utils/create_json_object.sql

@@ -159,4 +159,4 @@ select
     ) as {{ object_col_name }}
 from {{ table_ref }}
 group by {{ group_by_col }}
-{% endmacro %}
+{% endmacro %}

models/claims_preprocessing/claims_enrollment/claims_enrollment__flag_claims_with_enrollment.sql

@@ -60,7 +60,7 @@ select distinct
     , claim.inferred_claim_start_column_used
     , cast('{{ var('tuva_last_run') }}' as {{ dbt.type_timestamp() }}) as tuva_last_run
 from claim_year_month as claim
-inner join {{ ref('normalized_input__eligibility') }} as mm
+inner join {{ ref('claims_enrollment__member_months') }} as mm
     on mm.person_id = claim.person_id
     and mm.member_id = claim.member_id
     and mm.payer = claim.payer

models/claims_preprocessing/claims_enrollment/claims_enrollment__flag_rx_claims_with_enrollment.sql

@@ -39,7 +39,7 @@ select distinct
     , claim.paid_year_month
     , cast('{{ var('tuva_last_run') }}' as {{ dbt.type_timestamp() }}) as tuva_last_run
 from claim_dates as claim
-inner join {{ ref('normalized_input__eligibility') }} as mm
+inner join {{ ref('claims_enrollment__member_months') }} as mm
     on mm.person_id = claim.person_id
     and mm.member_id = claim.member_id
     and mm.payer = claim.payer

models/claims_preprocessing/claims_enrollment/claims_enrollment__member_months.sql

@@ -0,0 +1,49 @@
+{{ config(
+     enabled = var('claims_preprocessing_enabled',var('claims_enabled',var('tuva_marts_enabled',False)))
+ | as_bool
+   )
+}}
+
+
+with stg_eligibility as (
+  select
+    person_id
+    , member_id
+    , payer
+    , {{ quote_column('plan') }}
+    , data_source
+    , enrollment_start_date
+    , enrollment_end_date
+    , tuva_last_run
+  from {{ ref('normalized_input__eligibility') }} as elig
+)
+
+, month_start_and_end_dates as (
+  select
+    {{ concat_custom(["year",
+                  dbt.right(concat_custom(["'0'", "month"]), 2)]) }} as year_month
+    , min(full_date) as month_start_date
+    , max(full_date) as month_end_date
+  from {{ ref('reference_data__calendar') }}
+  group by year, month, year_month
+)
+select distinct
+  dense_rank() over (
+    order by
+      a.person_id
+    , b.year_month
+    , a.payer
+    , a.{{ quote_column('plan') }}
+    , a.data_source
+    ) as member_month_key
+  , a.person_id
+  , a.member_id
+  , b.year_month
+  , a.payer
+  , a.{{ quote_column('plan') }}
+  , a.data_source
+  , a.tuva_last_run
+from stg_eligibility as a
+inner join month_start_and_end_dates as b
+  on a.enrollment_start_date <= b.month_end_date
+  and a.enrollment_end_date >= b.month_start_date

models/claims_preprocessing/claims_preprocessing_models.yml

@@ -72,6 +72,35 @@ models:
       - name: paid_year_month
         description: the year and month of the date the pharmacy claim was paid.
 
+  - name: claims_enrollment__member_months
+    description: This table contains member months matching enrollment
+    config:
+      schema: |
+        {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_claims_preprocessing
+        {%- else -%}claims_preprocessing{%- endif -%}
+      alias: member_months
+      tags:
+        - claims_preprocessing
+      materialized: table
+    columns:
+      - name: member_month_key
+        description: Unique identifier for each member month.
+      - name: person_id
+        description: Unique identifier for each patient in the dataset.
+      - name: member_id
+        description: >
+          Identifier that links a patient to a particular insurance product or 
+          health plan. A patient can have more than one member_id because they 
+          can have more than one insurance product/plan.
+      - name: year_month
+        description: The year and month of the member month.
+      - name: payer
+        description: Name of the payer.
+      - name: plan
+        description: Name of the plan.
+      - name: data_source
+        description: '{{ doc("data_source") }}'
+
 
 
 ###   Normalized Claim Input Layer

models/claims_preprocessing/normalized_input/final/normalized_input__eligibility.sql

@@ -5,112 +5,36 @@
 }}
 
 
-with stg_eligibility as (
-  select
-    cast(elig.person_id as {{ dbt.type_string() }}) as person_id
-    , cast(elig.member_id as {{ dbt.type_string() }}) as member_id
-    , cast(elig.subscriber_id as {{ dbt.type_string() }}) as subscriber_id
-    , cast(elig.gender as {{ dbt.type_string() }}) as gender
-    , cast(elig.race as {{ dbt.type_string() }}) as race
-    , cast(date_norm.normalized_birth_date as date) as birth_date
-    , cast(date_norm.normalized_death_date as date) as death_date
-    , cast(elig.death_flag as int) as death_flag
-    , cast(date_norm.normalized_enrollment_start_date as date) as enrollment_start_date
-    , cast(date_norm.normalized_enrollment_end_date as date) as enrollment_end_date
-    , cast(elig.payer as {{ dbt.type_string() }}) as payer
-    , cast(elig.payer_type as {{ dbt.type_string() }}) as payer_type
-    , cast(elig.{{ quote_column('plan') }} as {{ dbt.type_string() }}) as {{ quote_column('plan') }}
-    , cast(elig.original_reason_entitlement_code as {{ dbt.type_string() }}) as original_reason_entitlement_code
-    , cast(elig.dual_status_code as {{ dbt.type_string() }}) as dual_status_code
-    , cast(elig.medicare_status_code as {{ dbt.type_string() }}) as medicare_status_code
-    , cast(elig.group_id as {{ dbt.type_string() }}) as group_id
-    , cast(elig.group_name as {{ dbt.type_string() }}) as group_name
-    , cast(elig.first_name as {{ dbt.type_string() }}) as first_name
-    , cast(elig.last_name as {{ dbt.type_string() }}) as last_name
-    , cast(elig.social_security_number as {{ dbt.type_string() }}) as social_security_number
-    , cast(elig.subscriber_relation as {{ dbt.type_string() }}) as subscriber_relation
-    , cast(elig.address as {{ dbt.type_string() }}) as address
-    , cast(elig.city as {{ dbt.type_string() }}) as city
-    , cast(elig.state as {{ dbt.type_string() }}) as state
-    , cast(elig.zip_code as {{ dbt.type_string() }}) as zip_code
-    , cast(elig.phone as {{ dbt.type_string() }}) as phone
-    , cast(elig.data_source as {{ dbt.type_string() }}) as data_source
-    , cast('{{ var('tuva_last_run') }}' as {{ dbt.type_string() }}) as tuva_last_run
-  from {{ ref('normalized_input__stg_eligibility') }} as elig
-  left outer join {{ ref('normalized_input__int_eligibility_dates_normalize') }} as date_norm
-    on elig.person_id_key = date_norm.person_id_key
-)
-
-, month_start_and_end_dates as (
-  select
-    {{ concat_custom(["year",
-                  dbt.right(concat_custom(["'0'", "month"]), 2)]) }} as year_month
-    , min(full_date) as month_start_date
-    , max(full_date) as month_end_date
-  from {{ ref('reference_data__calendar') }}
-  group by year, month, year_month
-)
-
-, member_month_calc as (
-  select distinct
-    dense_rank() over (
-      order by
-        a.person_id
-      , b.year_month
-      , a.payer
-      , a.{{ quote_column('plan') }}
-      , a.data_source
-      ) as member_month_key
-    , a.person_id
-    , b.year_month
-    , a.payer
-    , a.{{ quote_column('plan') }}
-    , a.data_source
-  from stg_eligibility as a
-  inner join month_start_and_end_dates as b
-    on a.enrollment_start_date <= b.month_end_date
-    and a.enrollment_end_date >= b.month_start_date
-)
-
 select
-    m.member_month_key
-  , a.person_id
-  , a.member_id
-  , a.subscriber_id
-  , a.gender
-  , a.race
-  , a.birth_date
-  , a.death_date
-  , a.death_flag
-  , a.enrollment_start_date
-  , a.enrollment_end_date
-  , a.payer
-  , a.payer_type
-  , a.{{ quote_column('plan') }}
-  , b.year_month
-  , a.original_reason_entitlement_code
-  , a.dual_status_code
-  , a.medicare_status_code
-  , a.group_id
-  , a.group_name
-  , a.first_name
-  , a.last_name
-  , a.social_security_number
-  , a.subscriber_relation
-  , a.address
-  , a.city
-  , a.state
-  , a.zip_code
-  , a.phone
-  , a.data_source
-  , a.tuva_last_run
-from stg_eligibility as a
-  inner join month_start_and_end_dates as b
-    on a.enrollment_start_date <= b.month_end_date
-    and a.enrollment_end_date >= b.month_start_date
-  inner join member_month_calc as m
-    on m.person_id = a.person_id
-    and m.payer = a.payer
-    and m.{{ quote_column('plan') }} = a.{{ quote_column('plan') }}
-    and m.data_source = a.data_source
-    and m.year_month = b.year_month
+  cast(elig.person_id as {{ dbt.type_string() }}) as person_id
+  , cast(elig.member_id as {{ dbt.type_string() }}) as member_id
+  , cast(elig.subscriber_id as {{ dbt.type_string() }}) as subscriber_id
+  , cast(elig.gender as {{ dbt.type_string() }}) as gender
+  , cast(elig.race as {{ dbt.type_string() }}) as race
+  , cast(date_norm.normalized_birth_date as date) as birth_date
+  , cast(date_norm.normalized_death_date as date) as death_date
+  , cast(elig.death_flag as int) as death_flag
+  , cast(date_norm.normalized_enrollment_start_date as date) as enrollment_start_date
+  , cast(date_norm.normalized_enrollment_end_date as date) as enrollment_end_date
+  , cast(elig.payer as {{ dbt.type_string() }}) as payer
+  , cast(elig.payer_type as {{ dbt.type_string() }}) as payer_type
+  , cast(elig.{{ quote_column('plan') }} as {{ dbt.type_string() }}) as {{ quote_column('plan') }}
+  , cast(elig.original_reason_entitlement_code as {{ dbt.type_string() }}) as original_reason_entitlement_code
+  , cast(elig.dual_status_code as {{ dbt.type_string() }}) as dual_status_code
+  , cast(elig.medicare_status_code as {{ dbt.type_string() }}) as medicare_status_code
+  , cast(elig.group_id as {{ dbt.type_string() }}) as group_id
+  , cast(elig.group_name as {{ dbt.type_string() }}) as group_name
+  , cast(elig.first_name as {{ dbt.type_string() }}) as first_name
+  , cast(elig.last_name as {{ dbt.type_string() }}) as last_name
+  , cast(elig.social_security_number as {{ dbt.type_string() }}) as social_security_number
+  , cast(elig.subscriber_relation as {{ dbt.type_string() }}) as subscriber_relation
+  , cast(elig.address as {{ dbt.type_string() }}) as address
+  , cast(elig.city as {{ dbt.type_string() }}) as city
+  , cast(elig.state as {{ dbt.type_string() }}) as state
+  , cast(elig.zip_code as {{ dbt.type_string() }}) as zip_code
+  , cast(elig.phone as {{ dbt.type_string() }}) as phone
+  , cast(elig.data_source as {{ dbt.type_string() }}) as data_source
+  , cast('{{ var('tuva_last_run') }}' as {{ dbt.type_string() }}) as tuva_last_run
+from {{ ref('normalized_input__stg_eligibility') }} as elig
+left outer join {{ ref('normalized_input__int_eligibility_dates_normalize') }} as date_norm
+  on elig.person_id_key = date_norm.person_id_key

models/core/core_models.yml

@@ -92,13 +92,16 @@ models:
       alias: eligibility
       tags: core
       materialized: table
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - eligibility_id
+            - data_source
     columns:
     - name: eligibility_id
       description: '{{ doc("eligibility_id") }}'
       tests:
-        - unique:
-            config:
-              severity: warn
+        - not_null
     - name: person_id
       description: '{{ doc("person_id") }}'
       tests:
@@ -143,6 +146,8 @@ models:
       description: '{{ doc("group_name") }}'
     - name: data_source
       description: '{{ doc("data_source") }}'
+      tests:
+        - not_null
     - name: tuva_last_run
       description: '{{ doc("tuva_last_run") }}'
 
@@ -704,6 +709,11 @@ models:
       tags:
       - core
       materialized: table
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - member_month_key
+            - data_source
     columns:
     - name: member_month_key
       description: '{{ doc("member_month_key") }}'
@@ -723,6 +733,8 @@ models:
       description: '{{ doc("plan") }}'
     - name: data_source
       description: '{{ doc("data_source") }}'
+      tests:
+        - not_null
     - name: tuva_last_run
       description: '{{ doc("tuva_last_run") }}'
     - name: payer_attributed_provider
@@ -1158,6 +1170,16 @@ models:
       tags: core_stage_claims
       materialized: table
 
+  - name: core__stg_claims_member_months
+    description: Staging core member months
+    config:
+      schema: |
+        {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_core
+        {% else %}core{%- endif -%}
+      alias: _stg_claims_member_months
+      tags: core_stage_claims
+      materialized: view
+  
   - name: core__stg_claims_patient
     description: Staging core claims patients
     config:

models/core/final/core__member_months.sql

@@ -3,63 +3,4 @@
    )
 }}
 
-with final_before_attribution_fields as (
-  select distinct
-      a.member_month_key
-    , a.person_id
-    , a.member_id
-    , a.year_month
-    , a.payer
-    , a.{{ quote_column('plan') }}
-    , data_source
-    , '{{ var('tuva_last_run') }}' as tuva_last_run
-  from {{ ref('normalized_input__eligibility') }} as a
-)
-
-, add_attribution_fields as (
-  select
-      a.member_month_key
-    , a.person_id
-    , a.member_id
-    , a.year_month
-    , a.payer
-    , a.{{ quote_column('plan') }}
-    , a.data_source
-    , a.tuva_last_run
-
-    , b.payer_attributed_provider
-    , b.payer_attributed_provider_practice
-    , b.payer_attributed_provider_organization
-    , b.payer_attributed_provider_lob
-    , b.custom_attributed_provider
-    , b.custom_attributed_provider_practice
-    , b.custom_attributed_provider_organization
-    , b.custom_attributed_provider_lob
-
-  from final_before_attribution_fields as a
-  left outer join {{ ref('financial_pmpm__stg_provider_attribution') }} as b
-  on a.person_id = b.person_id
-  and a.year_month = b.year_month
-  and a.payer = b.payer
-  and a.{{ quote_column('plan') }} = b.{{ quote_column('plan') }}
-  and a.data_source = b.data_source
-)
-
-select
-    member_month_key
-  , person_id
-  , member_id
-  , year_month
-  , payer
-  , {{ quote_column('plan') }}
-  , data_source
-  , tuva_last_run
-  , payer_attributed_provider
-  , payer_attributed_provider_practice
-  , payer_attributed_provider_organization
-  , payer_attributed_provider_lob
-  , custom_attributed_provider
-  , custom_attributed_provider_practice
-  , custom_attributed_provider_organization
-  , custom_attributed_provider_lob
-from add_attribution_fields
+select * from {{ ref('core__stg_claims_member_months') }}