This is a combination of 16 commits.

feat: new unit tests for dbt models and macros

.github/workflows/coverage.yml

@@ -1,6 +1,6 @@
 # Check documentation coverage
 
-name: dbt Docs Coverage
+name: dbt Tests & Coverage
 
 on:
   push:
@@ -17,7 +17,7 @@ env:
 
 jobs:
   build:
-    name: Build docs and check coverage
+    name: Check coverage & run tests
     runs-on: ubuntu-latest
     permissions:
       contents: "read"
@@ -46,8 +46,9 @@ jobs:
           tutor local do load-xapi-test-data
       - name: Check dbt tests
         run: |
-          dbt run
-          dbt test
+          mv unit-test-seeds ci-seeds
+          dbt build --full-refresh --selector all_tests
+          mv ci-seeds unit-test-seeds
       - name: Check docs coverage
         run: |
           dbt run

.gitignore

@@ -3,4 +3,4 @@ target/
 dbt_packages/
 logs/
 coverage.json
-
+*.DS_Store

README.rst

@@ -33,6 +33,15 @@ Running dbt
 
 ``dbt run`` will compile and create the models defined in the "aspects" dbt project. By default, dbt will look in the ``xapi`` schema to find source tables. The ``XAPI_SCHEMA`` environment variable can be used to specify a different schema.
 
+Testing
+*******
+
+``dbt test`` will run all data tests and any tests in the 'tests' directory (generic tests).
+``dbt test --selector ci_unit_tests`` will run all unit tests tagged with 'ci'. 
+- These require tables to be seeded first. To do this, add 'unit-test-seeds' to ``seed-paths:`` in ``dbt_project.yml`` and run ``dbt seed --full-refresh``.
+``dbt test --selector all_tests`` will run all data/generic/unit tests.
+
+
 More Help
 =========
 

dbt_project.yml

@@ -11,10 +11,10 @@ profile: "aspects"
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-model-paths: ["models"]
+model-paths: ["models",'ci-models']
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
-seed-paths: ["seeds"]
+seed-paths: ["seeds","ci-seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
@@ -34,3 +34,28 @@ models:
     # Config indicated by + and applies to all files under models/example/
     enrollment:
       +materialized: view
+
+# These are for unit test seeds. They will be used when 'unit-test-seeds' is added 
+# to seed-paths above or when CI tests run
+seeds:
+  aspects:
+    macros:
+      items_per_subsection_expected:
+        +column_types:
+          item_count: UInt64
+    base:
+      xapi_events_all_parsed_expected:
+        +column_types:
+          emission_time: DateTime64(6)
+    instance:
+      fact_instance_enrollments_expected:
+        +column_types:
+          course_enrollment_mode_status_cnt: UInt64
+    navigation:
+      fact_navigation_completion_expected:
+        +column_types:
+          page_count: UInt64
+    problems:
+      responses_expected:
+        +column_types:
+          first_success_at: Nullable(DateTime)

macros/format_float_value.sql

@@ -0,0 +1,7 @@
+{% macro format_float_value(value) %}
+
+    {% if target.name != "prod" %} CAST({{ value }} as Float32)
+    {% else %} {{ value }}
+    {% endif %}
+
+{% endmacro %}

macros/get_problem_id.sql

@@ -3,6 +3,6 @@
 -- or is followed by '/answer' or '/hint'
 {% macro get_problem_id(object_id) %}
     regexpExtract(
-        object_id, 'xblock/([\w\d-\+:@]*@problem\+block@[\w\d][^_]*)(_\d_\d)?', 1
+        object_id, 'xblock/([\w\d-\+:@]*@problem\+block@[\w\d][^_\/]*)(_\d_\d)?', 1
     )
 {% endmacro %}

macros/source_for_test.sql

@@ -0,0 +1,7 @@
+{% macro source_for_test(schema, table) %}
+
+    {% if target.name != "prod" %} {{ source(target.schema, table) }}
+    {% else %} {{ source(schema, table) }}
+    {% endif %}
+
+{% endmacro %}

models/base/schema.yml

@@ -5,29 +5,29 @@ models:
     description: "A materialized view for xAPI events"
     columns:
       - name: event_id
-        data_type: uuid
+        data_type: UUID
         description: "The unique identifier for the event"
       - name: verb_id
-        data_type: string
+        data_type: String
         description: "The xAPI verb identifier"
       - name: actor_id
-        data_type: string
+        data_type: String
         description: "The xAPI actor identifier"
       - name: object_id
-        data_type: string
+        data_type: String
         description: "The xAPI object identifier"
       - name: course_id
-        data_type: string
+        data_type: String
         description: "The fully-qualified course identifier URL"
       - name: course_key
         data_type: String
         description: "The course key for the course"
       - name: org
-        data_type: string
+        data_type: String
         description: "The organization that the course belongs to"
       - name: emission_time
-        data_type: datetime64(6)
+        data_type: DateTime64(6)
         description: "The time the event was emitted"
       - name: event
-        data_type: string
-        description: "The xAPI event as a string"
+        data_type: String
+        description: "The xAPI event as a String"

models/base/sources.yml

@@ -12,3 +12,11 @@ sources:
           - name: event_id
           - name: emission_time
           - name: event
+  - name: reporting
+    database: "reporting"
+    tables:
+      - name: xapi_events_all
+        columns:
+          - name: event_id
+          - name: emission_time
+          - name: event

models/base/unit_tests.yaml

@@ -0,0 +1,14 @@
+unit_tests:
+  - name: test_xapi_events_all_parsed
+    model: xapi_events_all_parsed
+    config:
+      tags: 'ci'
+    given:
+      - input: source("{{target.schema}}", "xapi_events_all")
+        format: sql
+        rows: |
+          select * from xapi_events_all
+    expect:
+      format: sql
+      rows: |
+        select * from xapi_events_all_parsed_expected

models/base/xapi_events_all_parsed.sql

@@ -52,4 +52,4 @@ select
     ) as org,
     emission_time as emission_time,
     event::String as event
-from {{ source("xapi", "xapi_events_all") }}
+from {{ source_for_test("xapi", "xapi_events_all") }}