Merge pull request #77 from fivetran/feature/standardized-billing-lin…

…e-item-model

Feature/standardized billing line item model

CHANGELOG.md

@@ -1,3 +1,12 @@
+# dbt_stripe_source v0.12.0
+
+[PR [#77](https://github.com/fivetran/dbt_stripe_source/pull/77)] includes the following updates:
+
+## Feature Updates
+- Addition of the following new staging models and accompanying upstream references:
+  - `stg_stripe__discount` (required for downstream `dbt_stripe` model transformations)
+  - `stg_stripe__product` (enabled by default, but can be disabled by setting the `stripe__using_subscriptions` variable to `false`)
+
 # dbt_stripe_source v0.11.1
 [PR [#74](https://github.com/fivetran/dbt_stripe_source/pull/74)] includes the following updates:
 

README.md

@@ -45,7 +45,7 @@ If you  are **not** using the [Stripe transformation package](https://github.com
 ```yaml
 packages:
   - package: fivetran/stripe_source
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `stripe` schema. If this is not where your stripe data is (for example, if your stripe schema is named `stripe_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -56,7 +56,7 @@ vars:
     stripe_schema: your_schema_name 
 ```
 ## Step 4: Disable models for non-existent sources
-This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, `subscription`, or `credit_note` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true` with the exception of `credit_note`. Add variables for only the tables you want to disable or enable respectively:
+This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, `subscription`, `product`, or `credit_note` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true` with the exception of `credit_note`. Add variables for only the tables you want to disable or enable respectively:
 
 ```yml
 # dbt_project.yml
@@ -65,7 +65,7 @@ This package takes into consideration that not every Stripe account utilizes the
 vars:
     stripe__using_invoices:        False  #Disable if you are not using the invoice and invoice_line_item tables
     stripe__using_payment_method:  False  #Disable if you are not using the payment_method and payment_method_card tables
-    stripe__using_subscriptions:   False  #Disable if you are not using the subscription and plan/price tables.
+    stripe__using_subscriptions:   False  #Disable if you are not using the subscription, product, and plan/price tables.
     stripe__using_credit_notes:    True   #Enable if you are using the credit note tables.
 
 ```

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'stripe_source'
-version: '0.11.1'
+version: '0.12.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
@@ -18,6 +18,7 @@ vars:
     credit_note: "{{ source('stripe', 'credit_note') }}"
     credit_note_line_item: "{{ source('stripe', 'credit_note_line_item') }}"
     customer: "{{ source('stripe', 'customer') }}"
+    discount: "{{ source('stripe', 'discount') }}"
     dispute: "{{ source('stripe', 'dispute') }}"
     fee: "{{ source('stripe', 'fee') }}"
     invoice: "{{ source('stripe', 'invoice') }}"
@@ -28,6 +29,7 @@ vars:
     payout: "{{ source('stripe', 'payout') }}"
     plan: "{{ source('stripe', 'plan') }}"
     price: "{{ source('stripe', 'price') }}"
+    product: "{{ source('stripe', 'product') }}"
     refund: "{{ source('stripe', 'refund') }}"
     subscription_history: "{{ source('stripe', 'subscription_history') }}"
     subscription: "{{ source('stripe', 'subscription') }}"

integration_tests/.gitignore

@@ -1,4 +1,4 @@
-
+package-lock.yml
 target/
 dbt_modules/
 logs/

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: stripe_source_integrations_tests_4
+      schema: stripe_source_integrations_tests_6
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: stripe_source_integrations_tests_4
+      schema: stripe_source_integrations_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: stripe_source_integrations_tests_4
+      schema: stripe_source_integrations_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: stripe_source_integrations_tests_4
+      schema: stripe_source_integrations_tests_6
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: stripe_source_integrations_tests_4
+      schema: stripe_source_integrations_tests_6
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,20 +1,21 @@
 config-version: 2
 
 name: 'stripe_source_integration_tests'
-version: '0.11.1'
+version: '0.12.0'
 
 
 profile: 'integration_tests'
 
 vars:
-  stripe_schema: stripe_source_integrations_tests_4
+  stripe_schema: stripe_source_integrations_tests_6
   stripe_source:
     stripe_account_identifier:                "account_data"
     stripe_group_identifier:                  "group_data"
     stripe_balance_transaction_identifier:    "balance_transaction_data"
     stripe_card_identifier:                   "card_data"
     stripe_charge_identifier:                 "charge_data"
     stripe_customer_identifier:               "customer_data"
+    stripe_discount_identifier:               "discount_data"
     stripe_dispute_identifier:                "dispute_data"
     stripe_fee_identifier:                    "fee_data"
     stripe_invoice_identifier:                "invoice_data"
@@ -24,6 +25,7 @@ vars:
     stripe_payment_method_identifier:         "payment_method_data"
     stripe_payout_identifier:                 "payout_data"
     stripe_plan_identifier:                   "plan_data"
+    stripe_product_identifier:                "product_data"
     stripe_price_identifier:                  "price_data"
     stripe_refund_identifier:                 "refund_data"
     stripe_subscription_history_identifier:   "subscription_history_data"
@@ -33,12 +35,13 @@ vars:
     stripe_credit_note_line_item_identifier:  "credit_note_line_item_data"
 
 seeds:
-  +quote_columns: "{{ true if target.type == 'redshift' else false }}"
+  +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
   stripe_source_integration_tests:
     +column_types:
       _fivetran_synced: timestamp
       created: timestamp
       voided_at: timestamp
+      end: timestamp
     account_data:
       +column_types:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
@@ -47,6 +50,15 @@ seeds:
     invoice_data:
       +column_types:
         id: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
+    discount_data:
+      +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
+    discount_data_snowflake:
+      +alias: discount_data
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+    discount_data_postgres:
+      +alias: discount_data
+      +enabled: "{{ true if target.type == 'postgres' else false }}"
+
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/discount_data.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,end,invoice_id,invoice_item_id,promotion_code,start,subscription_id
+one1,,cus_twelve,,1,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+2wo,,cus_twelve,,2,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+3hree,,cus_twelve,,3,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+4our,,cus_twelve,,4,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+5ive,,cus_twelve,,5,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+6ix,,cus_twelve,,6,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+7even,,cus_twelve,,7,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+8ight,,cus_twelve,,8,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+9ine,,cus_thirteen,,9,,,2023-08-07 09:37:39.000000,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000,
+10n,,cus_OPEKG00sHdsUs8,,10,,,2023-08-07 09:38:05.000000,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000,

integration_tests/seeds/discount_data_postgres.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,"end",invoice_id,invoice_item_id,promotion_code,"start",subscription_id
+one1,,cus_twelve,,1,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OvvZQDZlKRVG8GnByl08x0P,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+2wo,,cus_twelve,,2,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OlPk3DZlKRVG8GnkWTFA0go,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+3hree,,cus_twelve,,3,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OaAyXDZlKRVG8GnQo19CbEo,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+4our,,cus_twelve,,4,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+5ive,,cus_twelve,,5,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1P7ALFDZlKRVG8GnTUCtI40b,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+6ix,,cus_twelve,,6,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+7even,,cus_twelve,,7,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1PI2dvDZlKRVG8Gn4I34kEqR,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+8ight,,cus_twelve,,8,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+9ine,,cus_thirteen,,9,,,2023-08-07 09:37:39.000000,,cus_OPEJRxhitWa90P,"2022-09-27 05:50:07.000000",,,,"2023-08-07 09:37:39.000000",
+10n,,cus_OPEKG00sHdsUs8,,10,,,2023-08-07 09:38:05.000000,,cus_OPEKG00sHdsUs8,,,,,"2023-08-07 09:38:05.000000",

integration_tests/seeds/discount_data_snowflake.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,"""END""",invoice_id,invoice_item_id,promotion_code,"""START""",subscription_id
+one1,,cus_twelve,,1,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+2wo,,cus_twelve,,2,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+3hree,,cus_twelve,,3,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+4our,,cus_twelve,,4,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+5ive,,cus_twelve,,5,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+6ix,,cus_twelve,,6,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+7even,,cus_twelve,,7,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+8ight,,cus_twelve,,8,,,2022-09-19 05:50:07.000000,sub_nine,cus_MSXtGSAxxtYWe1,,,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+9ine,,cus_thirteen,,9,,,2023-08-07 09:37:39.000000,,cus_OPEJRxhitWa90P,,,,,"2023-08-07 09:37:39.000000",
+10n,,cus_OPEKG00sHdsUs8,,10,,,2023-08-07 09:38:05.000000,,cus_OPEKG00sHdsUs8,,,,,"2023-08-07 09:38:05.000000",

integration_tests/seeds/product_data.csv

@@ -0,0 +1,11 @@
+id,_fivetran_synced,active,caption,created,description,is_deleted,livemode,metadata,name,package_dimensions_height,package_dimensions_length,package_dimensions_weight,package_dimensions_width,shippable,statement_descriptor,type,unit_label,updated,url
+prod_one,2024-05-19 12:34:06.327000,true,,2022-09-19 05:49:41.000000,,false,true,{},Ultimiate Membership,,,,,,,service,,2022-09-19 05:49:41.000000,
+prod_two,2024-01-30 12:31:51.809000,true,Test,2024-01-30 06:18:03.000000,Test Description,false,true,{},Test Product OG,,,,,true,,good,,2024-01-30 06:22:27.000000,example.com
+prod_three,2024-02-27 12:32:13.504000,true,,2019-07-17 07:24:19.000000,,false,true,{},Test_product 1,,,,,,,service,,2022-05-05 19:13:59.000000,
+prod_four,2023-11-08 14:18:15.371000,true,,2023-06-08 11:20:00.000000,,false,true,{},zzzz,,,,,,,service,,2023-06-08 11:20:00.000000,
+prod_five,2024-06-05 12:31:29.500000,true,,2023-04-05 14:40:46.000000,,false,true,{},Test product for quanity,,,,,,,service,,2023-04-05 14:40:46.000000,
+prod_six,2022-11-16 20:28:40.972000,true,,2019-07-17 11:27:24.000000,,false,true,{},test_prod3,,,,,,,service,,2022-05-05 19:13:59.000000,
+prod_seven,2024-01-30 12:31:51.817000,true,,2024-01-30 07:18:38.000000,Black,false,true,{},Mobile,,,,,,,service,,2024-01-30 07:51:03.000000,
+prod_eight,2022-11-16 20:23:09.806000,true,,2022-07-12 09:22:46.000000,,false,true,{},test_product,,,,,,,service,,2022-07-12 09:22:46.000000,
+prod_nine,2023-12-05 08:18:45.455000,true,,2023-11-20 18:20:24.000000,,false,true,{},take 1,,,,,,,service,,2023-11-20 18:20:24.000000,
+prod_ten,2023-10-19 08:18:18.743000,true,,2023-04-12 07:23:48.000000,This is test product created for the tax rate testing,false,true,{},Tax Test,,,,,,,service,,2023-04-12 07:23:49.000000,

macros/get_discount_columns.sql

@@ -0,0 +1,30 @@
+{% macro get_discount_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_string() },
+    {"name": "type", "datatype": dbt.type_string() },
+    {"name": "type_id", "datatype": dbt.type_string() },
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp() },
+    {"name": "amount", "datatype": dbt.type_int() },
+    {"name": "checkout_session_id", "datatype": dbt.type_string() },
+    {"name": "checkout_session_line_item_id", "datatype": dbt.type_string() },
+    {"name": "coupon_id", "datatype": dbt.type_string() },
+    {"name": "credit_note_line_item_id", "datatype": dbt.type_string() },
+    {"name": "customer_id", "datatype": dbt.type_string() },
+    {"name": "invoice_id", "datatype": dbt.type_string() },
+    {"name": "invoice_item_id", "datatype": dbt.type_string() },
+    {"name": "promotion_code", "datatype": dbt.type_string() },
+    {"name": "subscription_id", "datatype": dbt.type_string() }
+] %}
+
+{% if target.type == 'snowflake' %}
+    {{ columns.append({"name": "END", "datatype": dbt.type_string(), "quote": True, "alias": "end_at"}) }},
+    {{ columns.append({"name": "START", "datatype": dbt.type_string(), "quote": True, "alias": "start_at"}) }}
+{% else %}
+    {{ columns.append({"name": "end", "datatype": dbt.type_string(), "quote": True, "alias": "end_at"}) }},
+    {{ columns.append({"name": "start", "datatype": dbt.type_string(), "quote": True, "alias": "start_at"}) }}
+{% endif %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_columns.sql

@@ -0,0 +1,26 @@
+{% macro get_product_columns() %}
+
+{% set columns = [
+    {"name": "fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "active", "datatype": "boolean","alias": "is_active"},
+    {"name": "attributes", "datatype": dbt.type_string()},
+    {"name": "caption", "datatype": dbt.type_string()},
+    {"name": "created", "datatype": dbt.type_timestamp()},
+    {"name": "deactivate_on", "datatype": dbt.type_timestamp()},
+    {"name": "description", "datatype": dbt.type_string()},
+    {"name": "images", "datatype":  dbt.type_string()},
+    {"name": "is_deleted", "datatype": "boolean"},
+    {"name": "livemode", "datatype": "boolean"},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "shippable", "datatype": "boolean"},
+    {"name": "statement_descriptor", "datatype": dbt.type_string()},
+    {"name": "type", "datatype": dbt.type_string()},
+    {"name": "unit_label", "datatype": dbt.type_string()},
+    {"name": "updated", "datatype": dbt.type_timestamp()},
+    {"name": "url", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}