Skip to content

Commit

Permalink
Update Tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilias1111
ilias1111 committed Nov 18, 2024
1 parent c8d21f3 commit 325d3dc
Show file tree
Hide file tree
Showing 11 changed files with 30 additions and 30 deletions.
8 changes: 4 additions & 4 deletions integration_tests/.scripts/integration_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,18 @@ for db in ${DATABASES[@]}; do
eval "dbt run --target $db --full-refresh" || exit 1
fi

echo "--- Snowplow normalize integration tests: snakeify case"
echo "Snowplow normalize integration tests: snakeify case"

eval "dbt run-operation test_snakeify_case --target $db" || exit 1;

echo "--- Snowplow normalize integration tests: normalize events"
echo "Snowplow normalize integration tests: normalize events"

eval "dbt run-operation test_normalize_events --target $db" || exit 1;

echo "--- Snowplow normalize integration tests: users table"
echo "Snowplow normalize integration tests: users table"

eval "dbt run-operation test_users_table --target $db" || exit 1;

echo "--- Snowplow normalize integration tests: All tests passed"
echo "Snowplow normalize integration tests: All tests passed"

done
2 changes: 1 addition & 1 deletion integration_tests/dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ vars:
snowplow__backfill_limit_days: 2
snowplow__derived_tstamp_partitioned: false
snowplow__atomic_schema: "{{ target.schema ~ 'sp_normalize_int_test' }}"
snowplow__partition_key: "collector_tstamp"
snowplow__partition_key: "load_tstamp"

models:
snowplow_normalize_integration_tests:
Expand Down
18 changes: 9 additions & 9 deletions integration_tests/macros/test_normalize_events.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,15 @@ It runs 9 tests:
{% macro databricks__test_normalize_events() %}
{% set expected_dict = {
"flat_cols_only" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_cols" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_cols_w_alias" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as my_alias_test_id , UNSTRUCT_EVENT_TEST_1.test_class as my_alias_test_class -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_1_context" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_2_context" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_2_context_w_alias" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as test1_context_test_id , CONTEXTS_TEST_1[0].context_test_class as test1_context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as test2_context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as test2_context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"context_only" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"multiple_base_events" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name','page_ping')",
"multiple_sde_events" : "select event_id , collector_tstamp , DATE(collector_tstamp) as collector_tstamp_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test1_test_id , UNSTRUCT_EVENT_TEST_1.test_class as test1_test_class , UNSTRUCT_EVENT_TEST2_1.test_word as test2_test_word , UNSTRUCT_EVENT_TEST2_1.test_idea as test2_test_idea -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')"
"flat_cols_only" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_cols" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_cols_w_alias" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as my_alias_test_id , UNSTRUCT_EVENT_TEST_1.test_class as my_alias_test_class -- context column(s) from the event table from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_1_context" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_2_context" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"sde_plus_2_context_w_alias" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test_id , UNSTRUCT_EVENT_TEST_1.test_class as test_class -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as test1_context_test_id , CONTEXTS_TEST_1[0].context_test_class as test1_context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as test2_context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as test2_context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"context_only" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')",
"multiple_base_events" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name','page_ping')",
"multiple_sde_events" : "select event_id , collector_tstamp , DATE(" ~ var('snowplow__partition_key') ~ ") as " ~ var('snowplow__partition_key') ~ "_date -- Flat columns from event table , app_id -- self describing events columns from event table , UNSTRUCT_EVENT_TEST_1.test_id as test1_test_id , UNSTRUCT_EVENT_TEST_1.test_class as test1_test_class , UNSTRUCT_EVENT_TEST2_1.test_word as test2_test_word , UNSTRUCT_EVENT_TEST2_1.test_idea as test2_test_idea -- context column(s) from the event table , CONTEXTS_TEST_1[0].context_test_id as context_test_id , CONTEXTS_TEST_1[0].context_test_class as context_test_class , CONTEXTS_TEST2_1[0].context_test_id2 as context_test_id2 , CONTEXTS_TEST2_1[0].context_test_class2 as context_test_class2 from `"~target.catalog~"`."~target.schema~"_scratch.snowplow_normalize_base_events_this_run where event_name in ('event_name')"
} %}
{% set results_dict ={
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name2_1.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name3_2.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name4_1.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name5_9.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name6_6.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/custom_table_name7_6.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/event_name1_1.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "event_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down
4 changes: 2 additions & 2 deletions utils/tests/expected/test_normalized_events.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
unique_key = "unique_id",
upsert_date_key = "collector_tstamp",
partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
"field": "collector_tstamp",
"field": var("snowplow__partition_key"),
"data_type": "timestamp"
}, databricks_val='collector_tstamp_date'),
}, databricks_val=databricks_partition()),
sql_header=snowplow_utils.set_query_tag(var('snowplow__query_tag', 'snowplow_dbt')),
tblproperties={
'delta.autoOptimize.optimizeWrite' : 'true',
Expand Down

0 comments on commit 325d3dc

Please sign in to comment.