Merge pull request #33 from fivetran/feature/api-updates

feature/api-updates

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

CHANGELOG.md

@@ -1,9 +1,52 @@
-# dbt_marketo_source v0.UPDATE.UPDATE
-
- ## Under the Hood:
+# dbt_marketo_source v0.10.0
+## 🚨 Breaking Changes 🚨:
+[PR #33](https://github.com/fivetran/dbt_marketo_source/pull/33) includes the following updates in connection with the Fivetran Marketo connector's [June 2023](https://fivetran.com/docs/applications/marketo/changelog#june2023) and [May 2023](https://fivetran.com/docs/applications/marketo/changelog#may2023) releases:
+- Added passthrough column support for the following sources. (**Note**: If you are missing any columns from the prior version of this package, utilize the passthrough capability to bring these columns in. Only non-custom columns are included going forward. For more information refer to the [Passing Through Additional Columns section](https://github.com/fivetran/dbt_marketo_source#optional-step-5-additional-configurations) in the README.)
+  - activity_send_email
+  - program
+- Following the connector upgrade, a number of fields were removed and added to the source tables. These added and removed fields have been accounted for appropriately within the staging models. See the full list below:
+  - Fields *added* to `lead`:
+    - email
+    - first_name
+    - last_name
+  - Fields *added* to `program`:
+    - _fivetran_deleted
+  - Fields *removed* from `campaign`:
+    - program_name
+  - Fields *added* to `campaign`:
+    - computed_url
+    - flow_id
+    - folder_id
+    - folder_type
+    - is_communication_limit_enabled
+    - is_requestable
+    - is_system
+    - max_members
+    - qualification_rule_type
+    - qualification_rule_interval
+    - qualification_rule_unit
+    - recurrence_start_at
+    - recurrence_end_at
+    - recurrence_interval_type
+    - recurrence_interval
+    - recurrence_weekday_only
+    - recurrence_day_of_month
+    - recurrence_day_of_week
+    - recurrence_week_of_month
+    - smart_list_id
+    - status
+    - _fivetran_deleted
 
+## 🚘 Under the Hood:
+[PR #33](https://github.com/fivetran/dbt_marketo_source/pull/33) includes the following updates:
+- Updated the following models to filter out records where `_fivetran_deleted`is true. 
+  - `stg_marketo__campaigns`
+  - `stg_marketo__program`
+- Updated documentation and testing seed data
+[PR #31](https://github.com/fivetran/dbt_marketo_source/pull/31) includes the following updates:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
+
 # dbt_marketo_source v0.9.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -43,7 +43,7 @@ If you  are **not** using the [Marketo transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/marketo_source
-    version: [">=0.9.0", "<0.10.0"]
+    version: [">=0.10.0", "<0.11.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -73,6 +73,20 @@ vars:
 <details><summary>Expand for details</summary>
 <br>
 
+### Passing Through Additional Columns
+This package includes all source columns defined in the macros folder. If you would like to pass through additional columns to the staging models, add the following configurations to your `dbt_project.yml` file. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables in your root `dbt_project.yml`.
+```yml
+vars:
+    marketo__activity_send_email_passthrough_columns: 
+      - name: "new_custom_field"
+        alias: "custom_field_name"
+        transform_sql:  "cast(custom_field_name as int64)"
+      - name: "a_second_field"
+        transform_sql:  "cast(a_second_field as string)"
+    # a similar pattern can be applied to the rest of the following variables.
+    marketo__program_passthrough_columns:
+```
+
 ### Changing the Build Schema
 By default this package will build the Marketo staging models within a schema titled (<target_schema> + `_marketo_source`) in your target database. If this is not where you would like your Marketo data to be written to, add the following configuration to your `dbt_project.yml` file:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'marketo_source'
-version: '0.9.0'
+version: '0.10.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -24,3 +24,5 @@ vars:
     activity_send_email: "{{ source('marketo','activity_send_email') }}"
     activity_delete_lead: "{{ source('marketo','activity_delete_lead') }}"
     activity_merge_leads: "{{ source('marketo','activity_merge_leads') }}"
+    marketo__activity_send_email_passthrough_columns: []
+    marketo__program_passthrough_columns: []

integration_tests/ci/sample.profiles.yml

@@ -49,6 +49,6 @@ integration_tests:
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: marketo_source_integration_tests_7
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'marketo_source_integration_tests'
-version: '0.9.0'
+version: '0.10.0'
 profile: 'integration_tests'
 config-version: 2
 vars:
@@ -20,7 +20,6 @@ vars:
     marketo_activity_delete_lead_identifier: "marketo_activity_delete_lead_data"
     marketo_activity_merge_leads_identifier: "marketo_activity_merge_leads_data"
 
-
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']