Merge pull request #1 from Unicon/feature/tpdm

Copy pull request edanalytics#84 from original repository.

dbt_project.yml

@@ -35,6 +35,12 @@ models:
            +tags: ['bypass_rls']
         stage: 
           +materialized: table
+      tpdm:
+        +enabled: "{{ var('edu:tpdm:enabled', false) }}"
+        base:
+          +materialized: view
+        stage:
+          +materialized: table
 
 vars:
   extensions: null

macros/gen_skey.sql

@@ -191,6 +191,17 @@
             'annualize': True
         },
 
+        'k_educator_prep_program': {
+            'reference_name': 'educator_preparation_program_reference',
+            'col_list': ['educationOrganizationId', 'programName', 'programTypeDescriptor'],
+            'annualize': True
+        },
+        'k_candidate': {
+            'reference_name': 'candidate_reference',
+            'col_list': ['candidateIdentifier'],
+            'annualize': True
+        },
+
         'k_template': {
             'reference_name': '',
             'col_list': [],

models/staging/tpdm/base/_tpdm__base.yml

@@ -0,0 +1,9 @@
+version: 2
+
+models:
+  - name: base_tpdm__candidates
+    config:
+      tags: ['tpdm']
+  - name: base_tpdm__educator_preparation_programs
+    config:
+      tags: ['tpdm']

models/staging/tpdm/base/base_tpdm__candidates.sql

@@ -0,0 +1,56 @@
+with candidates as (
+    {{ source_edfi3('candidates') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                         as record_guid,
+        v:candidateIdentifier::string        as candidate_id,
+        v:firstName::string                  as first_name,
+        v:lastSurname::string                as last_name,
+        v:middleName::string                 as middle_name,
+        v:maidenName::string                 as maiden_name,
+        v:generationCodeSuffix::string       as generation_code_suffix,
+        v:personalTitlePrefix::string        as personal_title_prefix,
+        v:preferredFirstName::string         as preferred_first_name,
+        v:preferredLastSurname::string       as preferred_last_name,
+        v:birthCity::string                  as birth_city,
+        v:birthDate::date                    as birth_date,
+        v:birthInternationalProvince::string as birth_international_province,
+        v:dateEnteredUS::date                as date_entered_us,
+        v:displacementStatus::string         as displacement_status,
+        v:economicDisadvantaged::boolean     as is_economic_disadvantaged,
+        v:firstGenerationStudent::boolean    as is_first_generation_student,
+        v:hispanicLatinoEthnicity::boolean   as has_hispanic_latino_ethnicity,
+        v:multipleBirthStatus::boolean       as is_multiple_birth,
+        -- descriptors
+        {{ extract_descriptor('v:genderDescriptor::string') }}                    as gender,
+        {{ extract_descriptor('v:sexDescriptor::string') }}                       as sex,
+        {{ extract_descriptor('v:birthSexDescriptor::string') }}                  as birth_sex,
+        {{ extract_descriptor('v:birthStateAbbreviationDescriptor::string') }}    as birth_state,
+        {{ extract_descriptor('v:birthCountyDescriptor::string') }}               as birth_county,
+        {{ extract_descriptor('v:englishLanguageExamDescriptor::string' )}}       as english_language_exam,
+        {{ extract_descriptor('v:limitedEnglishProficiencyDescriptor::string' )}} as lep_code,
+        -- unflattened lists
+        v:addresses                       as v_addresses,
+        v:disabilities                    as v_disabilities,
+        v:electronicMails                 as v_emails,
+        v:languages                       as v_languages,
+        v:otherNames                      as v_other_names,
+        v:personalIdentificationDocuments as v_personal_identification_documents,
+        v:races                           as v_races,
+        v:telephones                      as v_telephones,
+
+        -- references
+        v:personReference as person_reference,
+
+    from candidates
+)
+select * from renamed

models/staging/tpdm/base/base_tpdm__educator_preparation_programs.sql

@@ -0,0 +1,30 @@
+with educator_prep_programs as (
+    {{ source_edfi3('educator_preparation_programs') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string          as record_guid,
+        v:programId::string   as program_id,
+        v:programName::string as program_name,
+        v:educationOrganizationReference:educationOrganizationId::integer   as ed_org_id,
+        v:educationOrganizationReference:link:rel::string                   as ed_org_type,
+        -- descriptors
+        {{ extract_descriptor('v:programTypeDescriptor::string') }}         as program_type,
+        {{ extract_descriptor('v:accreditationStatusDescriptor::string') }} as accreditation_status,
+        -- unflattened lists
+        v:gradeLevels                    as v_grade_levels,
+        -- references
+        v:educationOrganizationReference as ed_org_reference,
+        -- edfi extensions
+        v:_ext                           as v_ext
+    from educator_prep_programs
+)
+select * from renamed

models/staging/tpdm/stage/_tpdm__stage.yml

@@ -0,0 +1,23 @@
+version: 2
+
+referential_integrity_tests:
+  - k_candidate: &ref_k_candidate
+      - relationships:
+          to: ref('stg_tpdm__candidates')
+          field: k_candidate
+          tags: ['ref_integrity']
+
+  - k_educator_prep_program: &ref_k_educator_prep_program
+      - relationships:
+          to: ref('stg_tpdm__educator_preparation_programs')
+          field: k_educator_prep_program
+          tags: ['ref_integrity']
+
+
+models:
+  - name: stg_tpdm__candidates
+    config:
+      tags: ['tpdm']
+  - name: stg_tpdm__educator_preparation_programs
+    config:
+      tags: ['tpdm']

models/staging/tpdm/stage/stg_tpdm__candidates.sql

@@ -0,0 +1,27 @@
+with base_candidates as (
+    select * from {{ ref('base_tpdm__candidates') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'api_year',
+                'lower(candidate_id)'
+            ]
+        ) }} as k_candidate,
+        base_candidates.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_candidates
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_candidate',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/tpdm/stage/stg_tpdm__educator_preparation_programs.sql

@@ -0,0 +1,30 @@
+with base_epp as (
+    select * from {{ ref('base_tpdm__educator_preparation_programs') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'api_year',
+                'lower(ed_org_id)',
+                'lower(program_name)',
+                'lower(program_type)'
+            ]
+        ) }} as k_educator_prep_program,
+        {{ edorg_ref(annualize=False) }},
+        base_epp.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_epp
+)
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_educator_prep_program',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped