Releases/v0.12.latest (#83)

* feat: add passthrough columns to card model. (#81) * feat: add non-standard columns to card model. * implement feedback from issue * Apply suggestions from code review Co-authored-by: Jamie Rodriguez <[email protected]> --------- Co-authored-by: Jamie Rodriguez <[email protected]> * polishes * docs * explain metadata better * link * try something out * revert * changelog * test out removing code block * revert and use yaml isntead of yml --------- Co-authored-by: bramrodenburg <[email protected]>
fivetran · Oct 2, 2024 · 4657edc · 4657edc
1 parent ed464b6
commit 4657edc
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,28 @@
+# dbt_stripe_source v0.12.1
+
+## Feature Updates
+- Declaration of passthrough variables for the `stg_stripe__card` model. This can, for example, be used to pull in non-standard columns from Stripe such as `description`, `iin` and `issuer`. See the [README](https://github.com/fivetran/dbt_stripe_source?tab=readme-ov-file#passing-through-additional-fields) for more details ([PR #81](https://github.com/fivetran/dbt_stripe_source/pull/81)).
+
+```yaml
+# dbt_project.yml
+
+vars:
+  stripe_source:
+    card_pass_through_columns:
+      - name: "description"
+      - name: "iin"
+      - name: "issuer"
+        alias: "card_issuer"  # optional: define an alias for the column 
+        transform_sql: "cast(card_issuer as string)" # optional: apply transformation to column. must reference the alias if provided
+```
+
+## Under the Hood
+- Removed un-used columns from the `get_card_columns()` [macro](https://github.com/fivetran/dbt_stripe_source/blob/main/macros/get_card_columns.sql) and aliased the `metadata` field, which is parsed via the `stripe__card_metadata` [variable](https://github.com/fivetran/dbt_stripe_source?tab=readme-ov-file#pivoting-out-metadata-properties), but not included as a field in its entirety in `stg_stripe__card` by default ([PR #83](https://github.com/fivetran/dbt_stripe_source/pull/83)).
+  - This ensures users can utilize the new `card_pass_through_columns` variable to include these fields in `stg_stripe__card`.
+
+## Contributors
+- [@bramrodenburg](https://github.com/bramrodenburg) ([#81](https://github.com/fivetran/dbt_stripe_source/pull/81))
+
 # dbt_stripe_source v0.12.0
 
 [PR [#77](https://github.com/fivetran/dbt_stripe_source/pull/77)] includes the following updates:

diff --git a/README.md b/README.md
@@ -178,6 +178,22 @@ vars:
     stripe__subscription_metadata: ['the', 'list', 'of', 'property', 'fields'] # Note: this is case-SENSITIVE and must match the casing of the property as it appears in the JSON
 ```
 
+#### Passing Through Additional Fields
+This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables:
+
+```yml
+# dbt_project.yml
+
+vars:
+  stripe_source:
+    card_pass_through_columns:
+      - name: "description"
+      - name: "iin"
+      - name: "issuer"
+        alias: "card_issuer"  # optional: define an alias for the column 
+        transform_sql: "cast(card_issuer as string)" # optional: apply transformation to column. must reference the alias if provided
+```
+
 #### Change the build schema
 By default, this package builds the stripe staging models within a schema titled (`<target_schema>` + `_stg_stripe`) in your destination. If this is not where you would like your stripe staging data to be written to, add the following configuration to your root `dbt_project.yml` file:
 
@@ -195,7 +211,7 @@ If an individual source table has a different name than the package expects, add
 vars:
     stripe_<default_source_table_name>_identifier: your_table_name 
 ```
-    
+
 </details>
 
 ### (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'stripe_source'
-version: '0.12.0'
+version: '0.12.1'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
@@ -33,4 +33,6 @@ vars:
     refund: "{{ source('stripe', 'refund') }}"
     subscription_history: "{{ source('stripe', 'subscription_history') }}"
     subscription: "{{ source('stripe', 'subscription') }}"
-    transfer: "{{ source('stripe', 'transfer') }}"
+    transfer: "{{ source('stripe', 'transfer') }}"
+
+    card_pass_through_columns: []
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,14 +1,24 @@
 config-version: 2
 
 name: 'stripe_source_integration_tests'
-version: '0.12.0'
+version: '0.12.1'
 
 
 profile: 'integration_tests'
 
 vars:
   stripe_schema: stripe_source_integrations_tests_6
+
   stripe_source:
+    card_pass_through_columns:
+      - name: "description"
+      - name: iin
+      - name: exp_year
+        alias: exp_year_alias
+        transform_sql: exp_year_alias * 2.0
+      - name: fingerprint
+      - name: metadata
+
     stripe_account_identifier:                "account_data"
     stripe_group_identifier:                  "group_data"
     stripe_balance_transaction_identifier:    "balance_transaction_data"

diff --git a/macros/get_card_columns.sql b/macros/get_card_columns.sql
@@ -6,34 +6,23 @@
     {"name": "address_city", "datatype": dbt.type_string()},
     {"name": "address_country", "datatype": dbt.type_string()},
     {"name": "address_line_1", "datatype": dbt.type_string()},
-    {"name": "address_line_1_check", "datatype": dbt.type_string()},
     {"name": "address_line_2", "datatype": dbt.type_string()},
     {"name": "address_state", "datatype": dbt.type_string()},
     {"name": "address_zip", "datatype": dbt.type_string()},
-    {"name": "address_zip_check", "datatype": dbt.type_string()},
     {"name": "brand", "datatype": dbt.type_string()},
-    {"name": "connected_account_id", "datatype": dbt.type_string()},
     {"name": "country", "datatype": dbt.type_string()},
     {"name": "created", "datatype": dbt.type_timestamp()},
-    {"name": "currency", "datatype": dbt.type_string()},
     {"name": "customer_id", "datatype": dbt.type_string()},
-    {"name": "cvc_check", "datatype": dbt.type_string()},
-    {"name": "dynamic_last_4", "datatype": dbt.type_string()},
-    {"name": "exp_month", "datatype": dbt.type_int()},
-    {"name": "exp_year", "datatype": dbt.type_int()},
-    {"name": "fingerprint", "datatype": dbt.type_string()},
     {"name": "funding", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_string()},
-    {"name": "is_deleted", "datatype": "boolean"},
-    {"name": "last_4", "datatype": dbt.type_string()},
-    {"name": "metadata", "datatype": dbt.type_string()},
+    {"name": "metadata", "datatype": dbt.type_string(), "alias": "card_metadata"},
     {"name": "name", "datatype": dbt.type_string()},
-    {"name": "network", "datatype": dbt.type_string()},
     {"name": "recipient", "datatype": dbt.type_string()},
-    {"name": "tokenization_method", "datatype": dbt.type_string()},
-    {"name": "wallet_type", "datatype": dbt.type_string()},
+    {"name": "wallet_type", "datatype": dbt.type_string()}
 ] %}
 
+{{ fivetran_utils.add_pass_through_columns(columns, var('card_pass_through_columns')) }}
+
 {{ return(columns) }}
 
 {% endmacro %}
diff --git a/models/stg_stripe__card.sql b/models/stg_stripe__card.sql
@@ -45,9 +45,11 @@ final as (
         source_relation
 
         {% if var('stripe__card_metadata',[]) %}
-        , {{ fivetran_utils.pivot_json_extract(string = 'metadata', list_of_properties = var('stripe__card_metadata')) }}
+        , {{ fivetran_utils.pivot_json_extract(string = 'card_metadata', list_of_properties = var('stripe__card_metadata')) }}
         {% endif %}
 
+        {{ fivetran_utils.fill_pass_through_columns('card_pass_through_columns') }}
+
     from fields
 )