From 62bab378b7cb791b34a6442ccbb240faad151c45 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:03:06 -0600 Subject: [PATCH 1/8] Create quickstart.yml --- .quickstart/quickstart.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .quickstart/quickstart.yml diff --git a/.quickstart/quickstart.yml b/.quickstart/quickstart.yml new file mode 100644 index 0000000..01a069a --- /dev/null +++ b/.quickstart/quickstart.yml @@ -0,0 +1,10 @@ +database_key: linkedin_pages_database +schema_key: linkedin_pages_schema + +dbt_versions: ">=1.3.0 <2.0.0" + +destination_configurations: + databricks: + dispatch: + - macro_namespace: dbt_utils + search_order: [ 'spark_utils', 'dbt_utils' ] From c3edfa80f067e2b06af6e77aedd8c5b723cb2992 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:15:16 -0600 Subject: [PATCH 2/8] Update pre-command --- .buildkite/hooks/pre-command | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..0e60322 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") From 6ae67267ed85a6afda0083af29d443c81cdf5c71 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:15:43 -0600 Subject: [PATCH 3/8] Update pipeline.yml --- .buildkite/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6e76ea0..552f698 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file + bash .buildkite/scripts/run_models.sh databricks From 991efb21eb1a49af2b7a576b864edb3a960cb887 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:16:15 -0600 Subject: [PATCH 4/8] Update sample.profiles.yml --- integration_tests/ci/sample.profiles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 7086930..de835c1 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -45,10 +45,10 @@ integration_tests: schema: linkedin_pages_integration_tests_1 threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: linkedin_pages_integration_tests_1 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file + type: databricks From 6bee59cd503e82a4d1c0b9248e41696e125fb64e Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:45:58 -0600 Subject: [PATCH 5/8] Update dbt_project.yml --- integration_tests/dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2f75ce2..fe163a7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -10,7 +10,7 @@ vars: linkedin_pages_post_content_identifier: "linkedin_pages_post_content_data" linkedin_pages_ugc_post_history_identifier: "linkedin_pages_ugc_post_history_data" linkedin_pages_ugc_post_share_statistic_identifier: "linkedin_pages_ugc_post_share_statistic_data" - linkedin_pages_schema: linkedin_pages_integration_tests_1 + linkedin_pages_schema: linkedin_pages_integration_tests_11 models: linkedin_pages_source: @@ -20,4 +20,4 @@ seeds: linkedin_pages_integration_tests: linkedin_pages_ugc_post_share_statistic_data: +column_types: - ugc_post_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" \ No newline at end of file + ugc_post_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" From 32be1f25e959e6d6ec934642b3df5d4acd174a19 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:46:15 -0600 Subject: [PATCH 6/8] Update sample.profiles.yml --- integration_tests/ci/sample.profiles.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index de835c1..28f8dbf 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: linkedin_pages_integration_tests_1 + schema: linkedin_pages_integration_tests_11 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: linkedin_pages_integration_tests_1 + schema: linkedin_pages_integration_tests_11 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: linkedin_pages_integration_tests_1 + schema: linkedin_pages_integration_tests_11 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: linkedin_pages_integration_tests_1 + schema: linkedin_pages_integration_tests_11 threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: linkedin_pages_integration_tests_1 + schema: linkedin_pages_integration_tests_11 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks From ede96ee2891a39443756c5e7155ad386f3b53087 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:57:04 -0600 Subject: [PATCH 7/8] Update requirements.txt --- integration_tests/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 42b50b8..0a6339c 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<1.5.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 - -oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file +dbt-databricks>=1.7.0,<2.0.0 From 0fecbbfb8a0383b79219d3fe3c7f9f76a4d6e10f Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:42:40 -0600 Subject: [PATCH 8/8] Update requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 0a6339c..a2ffbb8 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,6 +1,6 @@ dbt-snowflake>=1.3.0,<2.0.0 dbt-bigquery>=1.3.0,<2.0.0 -dbt-redshift>=1.3.0,<1.5.0 +dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0