From 63cd5e6239a5511f52c7ee9dbff565978f570200 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Mon, 17 Oct 2022 23:01:10 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Shopify:=20Add=20metafi?= =?UTF-8?q?eld=20streams=20(#17962)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎉 Source Shopify: Add metafield streams * Source Shopify: fix unittest * Source Shopify: docs update * Source Shopify: fix backward compatibility test * Source Shopify: fix schemas * Source Shopify: fix state filter * Source Shopify: refactor & optimize * Source Shopify: fix test privileges * Source Shopify: fix stream filter * Source Shopify: fix streams * Source Shopify: update abnormal state * Source Shopify: fix abnormal state streams * Source Shopify: fix streams * updated methods, formated code * Source Shopify: typo fix * auto-bump connector version Co-authored-by: Oleksandr Bazarnov Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-shopify/Dockerfile | 2 +- .../source-shopify/acceptance-test-config.yml | 6 + .../integration_tests/abnormal_state.json | 59 ++++- .../integration_tests/configured_catalog.json | 220 +++++++++++++++- .../integration_tests/integration_test.py | 8 - .../source_shopify/schemas/articles.json | 53 ++++ .../source_shopify/schemas/blogs.json | 44 ++++ .../source_shopify/schemas/collections.json | 46 ++++ .../schemas/metafield_articles.json | 46 ++++ .../{metafields.json => metafield_blogs.json} | 0 .../schemas/metafield_collections.json | 43 ++++ .../schemas/metafield_customers.json | 46 ++++ .../schemas/metafield_draft_orders.json | 46 ++++ .../schemas/metafield_locations.json | 46 ++++ .../schemas/metafield_orders.json | 46 ++++ .../schemas/metafield_pages.json | 46 ++++ .../schemas/metafield_product_images.json | 46 ++++ .../schemas/metafield_product_variants.json | 46 ++++ .../schemas/metafield_products.json | 46 ++++ .../schemas/metafield_shops.json | 46 ++++ .../schemas/metafield_smart_collections.json | 46 ++++ .../schemas/product_images.json | 37 +++ .../schemas/product_variants.json | 112 +++++++++ .../schemas/smart_collections.json | 49 ++++ .../source-shopify/source_shopify/source.py | 236 ++++++++++++++++-- .../source-shopify/source_shopify/utils.py | 35 ++- .../source-shopify/unit_tests/unit_test.py | 17 +- docs/integrations/sources/shopify.md | 83 +++--- 30 files changed, 1473 insertions(+), 87 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/integration_test.py create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json rename airbyte-integrations/connectors/source-shopify/source_shopify/schemas/{metafields.json => metafield_blogs.json} (100%) create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b1789a1f90b4..65f274bb60d4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -955,7 +955,7 @@ - name: Shopify sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.38 + dockerImageTag: 0.1.39 documentationUrl: https://docs.airbyte.com/integrations/sources/shopify icon: shopify.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e027db5e0214..bded7c7e5672 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10005,7 +10005,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-shopify:0.1.38" +- dockerImage: "airbyte/source-shopify:0.1.39" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/shopify" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 1e7a6898795c..62bbff5455f6 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.38 +LABEL io.airbyte.version=0.1.39 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index cda71978b962..5c542bfe96e0 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -17,8 +17,14 @@ tests: status: "failed" discovery: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.38" - config_path: "secrets/config_old.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.38" - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.38" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index e9e8facca510..73502ba27e01 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -1,7 +1,22 @@ { + "articles": { + "id": 99999999999999 + }, + "metafield_articles": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "blogs": { + "id": 99999999999999 + }, + "metafield_blogs": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "customers": { "updated_at": "2025-07-08T05:40:38-07:00" }, + "metafield_customers": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "orders": { "updated_at": "2025-07-08T05:40:38-07:00" }, @@ -14,15 +29,48 @@ "abandoned_checkouts": { "updated_at": "2025-07-08T05:40:38-07:00" }, - "metafields": { + "metafield_draft_orders": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "metafield_orders": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "product_images": { + "id": 99999999999999 + }, + "metafield_product_images": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "product_variants": { + "id": 99999999999999 + }, + "metafield_product_variants": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "metafield_products": { "updated_at": "2025-07-08T05:40:38-07:00" }, "collects": { "id": 29427031703741 }, + "collections": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "metafield_collections": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "smart_collections": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, + "metafield_smart_collections": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "custom_collections": { "updated_at": "2025-07-08T05:40:38-07:00" }, + "metafield_custom_collections": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "order_refunds": { "created_at": "2025-03-03T03:47:46-08:00", "orders": { @@ -35,6 +83,9 @@ "updated_at": "2025-02-22T00:37:28-08:00" } }, + "metafield_locations": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "transactions": { "created_at": "2025-03-03T03:47:45-08:00", "orders": { @@ -47,6 +98,9 @@ "pages": { "updated_at": "2025-07-08T05:24:10-07:00" }, + "metafield_pages": { + "updated_at": "2025-07-08T05:40:38-07:00" + }, "price_rules": { "updated_at": "2025-09-10T06:48:10-07:00" }, @@ -80,5 +134,8 @@ }, "balance_transactions": { "id": 9999999999999 + }, + "metafield_shops": { + "updated_at": "2025-07-08T05:40:38-07:00" } } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index be0c8854c55c..0ea5a78e05e6 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -1,5 +1,53 @@ { "streams": [ + { + "stream": { + "name": "articles", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_articles", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "blogs", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_blogs", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "customers", @@ -12,6 +60,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "metafield_customers", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "orders", @@ -24,6 +84,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "metafield_orders", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "draft_orders", @@ -36,6 +108,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "metafield_draft_orders", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "products", @@ -50,7 +134,31 @@ }, { "stream": { - "name": "abandoned_checkouts", + "name": "metafield_products", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "product_images", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_product_images", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, @@ -62,7 +170,31 @@ }, { "stream": { - "name": "metafields", + "name": "product_variants", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_product_variants", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "abandoned_checkouts", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, @@ -84,6 +216,30 @@ "cursor_field": ["id"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "collections", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_collections", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "custom_collections", @@ -96,6 +252,30 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "smart_collections", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafield_smart_collections", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "order_refunds", @@ -156,6 +336,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "metafield_pages", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "price_rules", @@ -178,6 +370,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "metafield_shops", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "discount_codes", @@ -202,6 +406,18 @@ "cursor_field": ["id"], "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "metafield_locations", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "inventory_items", diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-shopify/integration_tests/integration_test.py deleted file mode 100644 index 869f9c8fc10e..000000000000 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/integration_test.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -def test_dummy_test(): - """this is the dummy test to pass integration tests step""" - pass diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json new file mode 100644 index 000000000000..f1b09743f014 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "blog_id": { + "type": ["null", "integer"] + }, + "author": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + }, + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "summary_html": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + }, + "handle": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json new file mode 100644 index 000000000000..003a867b5672 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "commentable": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "feedburner": { + "type": ["null", "string"], + "format": "date-time" + }, + "feedburner_location": { + "type": ["null", "integer"] + }, + "handle": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "tags": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json new file mode 100644 index 000000000000..d2503f842f83 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json @@ -0,0 +1,46 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "handle": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "sort_order": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + }, + "products_count": { + "type": ["null", "integer"] + }, + "collection_type": { + "type": ["null", "string"] + }, + "published_scope": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json new file mode 100644 index 000000000000..1e91c726368f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json similarity index 100% rename from airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json rename to airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json new file mode 100644 index 000000000000..3a4a7bc1fcbc --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json @@ -0,0 +1,43 @@ +{ + "properties": { + "owner_id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "owner_resource": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json new file mode 100644 index 000000000000..1e91c726368f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json new file mode 100644 index 000000000000..1e91c726368f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json new file mode 100644 index 000000000000..bf74f22f5d1f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_resource": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json new file mode 100644 index 000000000000..1b9299c3976e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json @@ -0,0 +1,37 @@ +{ + "type": ["null", "object"], + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "src": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json new file mode 100644 index 000000000000..c5cac4d7e253 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json @@ -0,0 +1,112 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "number"], + "format": "float" + }, + "sku": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "inventory_policy": { + "type": ["null", "string"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "inventory_management": { + "type": ["null", "string"] + }, + "option1": { + "type": ["null", "string"] + }, + "option2": { + "type": ["null", "string"] + }, + "option3": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "taxable": { + "type": ["null", "boolean"] + }, + "barcode": { + "type": ["null", "string"] + }, + "grams": { + "type": ["null", "integer"] + }, + "image_id": { + "type": ["null", "integer"] + }, + "weight": { + "type": ["null", "number"], + "format": "float" + }, + "weight_unit": { + "type": ["null", "string"] + }, + "inventory_item_id": { + "type": ["null", "integer"] + }, + "inventory_quantity": { + "type": ["null", "integer"] + }, + "old_inventory_quantity": { + "type": ["null", "integer"] + }, + "presentment_prices": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "compare_at_price": { + "type": ["null", "number"] + } + } + } + }, + "requires_shipping" : { + "type" : ["null", "boolean"] + }, + "admin_graphql_api_id" : { + "type" : ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json new file mode 100644 index 000000000000..01930de03b54 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json @@ -0,0 +1,49 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "handle": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "sort_order": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + }, + "disjunctive": { + "type": ["null", "boolean"] + }, + "rules" : { + "type" : ["null", "array"], + "items" : { + "type" : ["null", "string"] + } + }, + "published_scope" : { + "type" : ["null", "string"] + }, + "admin_graphql_api_id" : { + "type" : ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 2e8dbe03958c..700d9113bd1b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -67,7 +67,7 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> @limiter.balance_rate_limit() def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() + json_response = response.json() or {} records = json_response.get(self.data_field, []) if self.data_field is not None else json_response # transform method was implemented according to issue 4841 # Shopify API returns price fields as a string and it should be converted to number @@ -88,6 +88,7 @@ def should_retry(self, response: requests.Response) -> bool: if response.status_code == 404: self.logger.warn(f"Stream `{self.name}` is not available, skipping.") setattr(self, "raise_on_http_errors", False) + return False return super().should_retry(response) @property @@ -136,14 +137,23 @@ def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Iterable[Mapping] = None) -> Iterable: # Getting records >= state if stream_state: + state_value = stream_state.get(self.cursor_field) for record in records_slice: if self.cursor_field in record: - if record.get(self.cursor_field, self.default_state_comparison_value) >= stream_state.get(self.cursor_field): + record_value = record.get(self.cursor_field, self.default_state_comparison_value) + if record_value: + if record_value >= state_value: + yield record + else: + # old entities could have cursor field in place, but set to null + self.logger.warn( + f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` cursor value is: {record_value}, record is emitted without state comparison" + ) yield record else: # old entities could miss the cursor field self.logger.warn( - f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` missing cursor: {self.cursor_field}" + f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` missing cursor field: {self.cursor_field}, record is emitted without state comparison" ) yield record else: @@ -171,6 +181,7 @@ class ShopifySubstream(IncrementalShopifyStream): nested_record: str = "id" nested_record_field_name: str = None nested_substream = None + nested_substream_list_field_id = None @property def parent_stream(self) -> object: @@ -232,7 +243,20 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite # to limit the number of API Calls and reduce the time of data fetch, # we can pull the ready data for child_substream, if nested data is present, # and corresponds to the data of child_substream we need. - if self.nested_substream: + if self.nested_substream and self.nested_substream_list_field_id: + if record.get(self.nested_substream): + sorted_substream_slices.extend( + [ + { + self.slice_key: sub_record[self.nested_substream_list_field_id], + self.cursor_field: record[self.nested_substream][0].get( + self.cursor_field, self.default_state_comparison_value + ), + } + for sub_record in record[self.nested_record] + ] + ) + elif self.nested_substream: if record.get(self.nested_substream): sorted_substream_slices.append( { @@ -270,6 +294,45 @@ def read_records( yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=records) +class MetafieldShopifySubstream(ShopifySubstream): + slice_key = "id" + data_field = "metafields" + + parent_stream_class: object = None + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + object_id = stream_slice[self.slice_key] + return f"{self.parent_stream_class.data_field}/{object_id}/{self.data_field}.json" + + +class Articles(IncrementalShopifyStream): + data_field = "articles" + cursor_field = "id" + order_field = "id" + filter_field = "since_id" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class MetafieldArticles(MetafieldShopifySubstream): + parent_stream_class: object = Articles + + +class Blogs(IncrementalShopifyStream): + cursor_field = "id" + order_field = "id" + data_field = "blogs" + filter_field = "since_id" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class MetafieldBlogs(MetafieldShopifySubstream): + parent_stream_class: object = Blogs + + class Customers(IncrementalShopifyStream): data_field = "customers" @@ -277,6 +340,10 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldCustomers(MetafieldShopifySubstream): + parent_stream_class: object = Customers + + class Orders(IncrementalShopifyStream): data_field = "orders" @@ -292,6 +359,10 @@ def request_params( return params +class MetafieldOrders(MetafieldShopifySubstream): + parent_stream_class: object = Orders + + class DraftOrders(IncrementalShopifyStream): data_field = "draft_orders" @@ -299,13 +370,72 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldDraftOrders(MetafieldShopifySubstream): + parent_stream_class: object = DraftOrders + + class Products(IncrementalShopifyStream): + use_cache = True data_field = "products" def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldProducts(MetafieldShopifySubstream): + parent_stream_class: object = Products + + +class ProductImages(ShopifySubstream): + parent_stream_class: object = Products + cursor_field = "id" + slice_key = "product_id" + data_field = "images" + nested_substream = "images" + filter_field = "since_id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + product_id = stream_slice[self.slice_key] + return f"products/{product_id}/{self.data_field}.json" + + +class MetafieldProductImages(MetafieldShopifySubstream): + parent_stream_class: object = Products + nested_record = "images" + slice_key = "images" + nested_substream = "images" + nested_substream_list_field_id = "id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + image_id = stream_slice[self.slice_key] + return f"product_images/{image_id}/{self.data_field}.json" + + +class ProductVariants(ShopifySubstream): + parent_stream_class: object = Products + cursor_field = "id" + slice_key = "product_id" + data_field = "variants" + nested_substream = "variants" + filter_field = "since_id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + product_id = stream_slice[self.slice_key] + return f"products/{product_id}/{self.data_field}.json" + + +class MetafieldProductVariants(MetafieldShopifySubstream): + parent_stream_class: object = Products + nested_record = "variants" + slice_key = "variants" + nested_substream = "variants" + nested_substream_list_field_id = "id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + variant_id = stream_slice[self.slice_key] + return f"variants/{variant_id}/{self.data_field}.json" + + class AbandonedCheckouts(IncrementalShopifyStream): data_field = "checkouts" @@ -322,20 +452,24 @@ def request_params( return params -class Metafields(IncrementalShopifyStream): - data_field = "metafields" +class CustomCollections(IncrementalShopifyStream): + data_field = "custom_collections" def path(self, **kwargs) -> str: return f"{self.data_field}.json" -class CustomCollections(IncrementalShopifyStream): - data_field = "custom_collections" +class SmartCollections(IncrementalShopifyStream): + data_field = "smart_collections" def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldSmartCollections(MetafieldShopifySubstream): + parent_stream_class: object = SmartCollections + + class Collects(IncrementalShopifyStream): """ Collects stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: @@ -355,6 +489,27 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class Collections(ShopifySubstream): + parent_stream_class: object = Collects + nested_record = "collection_id" + slice_key = "collection_id" + data_field = "collection" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + collection_id = stream_slice[self.slice_key] + return f"collections/{collection_id}.json" + + +class MetafieldCollections(MetafieldShopifySubstream): + parent_stream_class: object = Collects + slice_key = "collection_id" + nested_record = "collection_id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + object_id = stream_slice[self.slice_key] + return f"collections/{object_id}/{self.data_field}.json" + + class BalanceTransactions(IncrementalShopifyStream): """ @@ -422,6 +577,10 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldPages(MetafieldShopifySubstream): + parent_stream_class: object = Pages + + class PriceRules(IncrementalShopifyStream): data_field = "price_rules" @@ -453,6 +612,10 @@ def path(self, **kwargs): return f"{self.data_field}.json" +class MetafieldLocations(MetafieldShopifySubstream): + parent_stream_class: object = Locations + + class InventoryLevels(ShopifySubstream): parent_stream_class: object = Locations slice_key = "location_id" @@ -513,6 +676,13 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class MetafieldShops(IncrementalShopifyStream): + data_field = "metafields" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + class SourceShopify(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ @@ -535,7 +705,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ config["authenticator"] = ShopifyAuthenticator(config) user_scopes = self.get_user_scopes(config) - always_permitted_streams = ["Metafields", "Shop"] + always_permitted_streams = ["MetafieldShops", "Shop"] permitted_streams = [ stream for user_scope in user_scopes @@ -545,28 +715,46 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # before adding stream to stream_instances list, please add it to SCOPES_MAPPING stream_instances = [ - Customers(config), - Orders(config), - DraftOrders(config), - Products(config), AbandonedCheckouts(config), - Metafields(config), - CustomCollections(config), + Articles(config), + BalanceTransactions(config), + Blogs(config), + Collections(config), Collects(config), + CustomCollections(config), + Customers(config), + DiscountCodes(config), + DraftOrders(config), + FulfillmentOrders(config), + Fulfillments(config), + InventoryItems(config), + InventoryLevels(config), + Locations(config), + MetafieldArticles(config), + MetafieldBlogs(config), + MetafieldCollections(config), + MetafieldCustomers(config), + MetafieldDraftOrders(config), + MetafieldLocations(config), + MetafieldOrders(config), + MetafieldPages(config), + MetafieldProductImages(config), + MetafieldProducts(config), + MetafieldProductVariants(config), + MetafieldShops(config), + MetafieldSmartCollections(config), OrderRefunds(config), OrderRisks(config), - TenderTransactions(config), - Transactions(config), - BalanceTransactions(config), + Orders(config), Pages(config), PriceRules(config), - DiscountCodes(config), - Locations(config), - InventoryItems(config), - InventoryLevels(config), - FulfillmentOrders(config), - Fulfillments(config), + ProductImages(config), + Products(config), + ProductVariants(config), Shop(config), + SmartCollections(config), + TenderTransactions(config), + Transactions(config), ] return [stream_instance for stream_instance in stream_instances if self.format_name(stream_instance.name) in permitted_streams] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index 8e3eab27d401..aa349f3e7a07 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -10,17 +10,40 @@ import requests SCOPES_MAPPING = { - "read_customers": ["Customers"], - "read_orders": ["Orders", "AbandonedCheckouts", "TenderTransactions", "Transactions", "Fulfillments", "OrderRefunds", "OrderRisks"], - "read_draft_orders": ["DraftOrders"], - "read_products": ["Products", "CustomCollections", "Collects"], - "read_content": ["Pages"], + "read_customers": ["Customers", "MetafieldCustomers"], + "read_orders": [ + "Orders", + "AbandonedCheckouts", + "TenderTransactions", + "Transactions", + "Fulfillments", + "OrderRefunds", + "OrderRisks", + "MetafieldOrders", + ], + "read_draft_orders": ["DraftOrders", "MetafieldDraftOrders"], + "read_products": [ + "Products", + "MetafieldProducts", + "ProductImages", + "MetafieldProductImages", + "MetafieldProductVariants", + "CustomCollections", + "Collects", + "Collections", + "ProductVariants", + "MetafieldCollections", + "SmartCollections", + "MetafieldSmartCollections", + ], + "read_content": ["Pages", "MetafieldPages"], "read_price_rules": ["PriceRules"], "read_discounts": ["DiscountCodes"], - "read_locations": ["Locations"], + "read_locations": ["Locations", "MetafieldLocations"], "read_inventory": ["InventoryItems", "InventoryLevels"], "read_merchant_managed_fulfillment_orders": ["FulfillmentOrders"], "read_shopify_payments_payouts": ["BalanceTransactions"], + "read_online_store_pages": ["Articles", "MetafieldArticles", "Blogs", "MetafieldBlogs"], } diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index 84684c28a032..2c223d2117eb 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -32,15 +32,16 @@ def test_privileges_validation(requests_mock, basic_config): source = SourceShopify() expected = [ - "orders", "abandoned_checkouts", - "metafields", + "fulfillments", + "metafield_orders", + "metafield_shops", "order_refunds", "order_risks", + "orders", + "shop", "tender_transactions", "transactions", - "fulfillments", - "shop", ] assert [stream.name for stream in source.streams(basic_config)] == expected @@ -66,9 +67,15 @@ def test_filter_records_newer_than_state(basic_config): {"id": 1, "updated_at": "2022-01-01T01:01:01-07:00"}, # missing cursor, record should be present {"id": 2}, + # cursor is set to null + {"id": 3, "updated_at": "null"}, + # cursor is set to null + {"id": 4, "updated_at": None}, ] state = {"updated_at": "2022-02-01T00:00:00-07:00"} - expected = [{"id": 2}] + # we expect records with: id = 2, 3, 4. We output them `As Is`, + # because we cannot compare them to the STATE and SKIPPING them leads to data loss, + expected = [{"id": 2}, {"id": 3, "updated_at": "null"}, {"id": 4, "updated_at": None}] result = list(stream.filter_records_newer_than_state(state, records_slice)) assert result == expected diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index a54b834d66a1..da8df47995ec 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -94,9 +94,13 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica This Source is capable of syncing the following core Streams: +* [Articles](https://shopify.dev/api/admin-rest/2022-01/resources/article) +* [Blogs](https://shopify.dev/api/admin-rest/2022-01/resources/blog) * [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts#top) * [Collects](https://shopify.dev/api/admin-rest/2022-01/resources/collect#top) +* [Collections](https://shopify.dev/api/admin-rest/2022-01/resources/collection) * [Custom Collections](https://shopify.dev/api/admin-rest/2022-01/resources/customcollection#top) +* [Smart Collections](https://shopify.dev/api/admin-rest/2022-01/resources/smartcollection) * [Customers](https://shopify.dev/api/admin-rest/2022-01/resources/customer#top) * [Draft Orders](https://shopify.dev/api/admin-rest/2022-01/resources/draftorder#top) * [Discount Codes](https://shopify.dev/api/admin-rest/2022-01/resources/discountcode#top) @@ -105,6 +109,8 @@ This Source is capable of syncing the following core Streams: * [Orders Refunds](https://shopify.dev/api/admin-rest/2022-01/resources/refund#top) * [Orders Risks](https://shopify.dev/api/admin-rest/2022-01/resources/order-risk#top) * [Products](https://shopify.dev/api/admin-rest/2022-01/resources/product#top) +* [Product Images](https://shopify.dev/api/admin-rest/2022-01/resources/product-image) +* [Product Variants](https://shopify.dev/api/admin-rest/2022-01/resources/product-variant) * [Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/transaction#top) * [Tender Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/tendertransaction) * [Pages](https://shopify.dev/api/admin-rest/2022-01/resources/page#top) @@ -137,41 +143,42 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.1.38 | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777) | Fixed `404` for configured streams, fix missing `cursor` error for old records -| 0.1.37 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.36 | 2022-03-22 | [9850](https://github.com/airbytehq/airbyte/pull/9850) | Added `BalanceTransactions` stream | -| 0.1.35 | 2022-03-07 | [10915](https://github.com/airbytehq/airbyte/pull/10915) | Fix a bug which caused `full-refresh` syncs of child REST entities configured for `incremental` | -| 0.1.34 | 2022-03-02 | [10794](https://github.com/airbytehq/airbyte/pull/10794) | Minor specification re-order, fixed links in documentation | -| 0.1.33 | 2022-02-17 | [10419](https://github.com/airbytehq/airbyte/pull/10419) | Fixed wrong field type for tax_exemptions for `Abandoned_checkouts` stream | -| 0.1.32 | 2022-02-18 | [10449](https://github.com/airbytehq/airbyte/pull/10449) | Added `tender_transactions` stream | -| 0.1.31 | 2022-02-08 | [10175](https://github.com/airbytehq/airbyte/pull/10175) | Fixed compatibility issues for legacy user config | -| 0.1.30 | 2022-01-24 | [9648](https://github.com/airbytehq/airbyte/pull/9648) | Added permission validation before sync | -| 0.1.29 | 2022-01-20 | [9049](https://github.com/airbytehq/airbyte/pull/9248) | Added `shop_url` to the record for all streams | -| 0.1.28 | 2022-01-19 | [9591](https://github.com/airbytehq/airbyte/pull/9591) | Implemented `OAuth2.0` authentication method for Airbyte Cloud | -| 0.1.27 | 2021-12-22 | [9049](https://github.com/airbytehq/airbyte/pull/9049) | Update connector fields title/description | -| 0.1.26 | 2021-12-14 | [8597](https://github.com/airbytehq/airbyte/pull/8597) | Fix `mismatched number of tables` for base-normalization, increased performance of `order_refunds` stream | -| 0.1.25 | 2021-12-02 | [8297](https://github.com/airbytehq/airbyte/pull/8297) | Added Shop stream | -| 0.1.24 | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783) | Reviewed and corrected schemas for all streams | -| 0.1.23 | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973) | Added `InventoryItems` | -| 0.1.22 | 2021-10-18 | [7101](https://github.com/airbytehq/airbyte/pull/7107) | Added FulfillmentOrders, Fulfillments streams | -| 0.1.21 | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382) | Fixed `InventoryLevels` primary key | -| 0.1.20 | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063) | Added `Location` and `InventoryLevels` as streams | -| 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option | -| 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema | -| 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers | -| 0.1.16 | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945) | Fixed the connector's performance for `Incremental refresh` | -| 0.1.15 | 2021-09-02 | [5853](https://github.com/airbytehq/airbyte/pull/5853) | Fixed `amount` type in `order_refund` schema | -| 0.1.14 | 2021-09-02 | [5801](https://github.com/airbytehq/airbyte/pull/5801) | Fixed `line_items/discount allocations` & `duties` parts of `orders` schema | -| 0.1.13 | 2021-08-17 | [5470](https://github.com/airbytehq/airbyte/pull/5470) | Fixed rate limits throttling | -| 0.1.12 | 2021-08-09 | [5276](https://github.com/airbytehq/airbyte/pull/5276) | Add status property to product schema | -| 0.1.11 | 2021-07-23 | [4943](https://github.com/airbytehq/airbyte/pull/4943) | Fix products schema up to API 2021-07 | -| 0.1.10 | 2021-07-19 | [4830](https://github.com/airbytehq/airbyte/pull/4830) | Fix for streams json schemas, upgrade to API version 2021-07 | -| 0.1.9 | 2021-07-04 | [4472](https://github.com/airbytehq/airbyte/pull/4472) | Incremental sync is now using updated\_at instead of since\_id by default | -| 0.1.8 | 2021-06-29 | [4121](https://github.com/airbytehq/airbyte/pull/4121) | Add draft orders stream | -| 0.1.7 | 2021-06-26 | [4290](https://github.com/airbytehq/airbyte/pull/4290) | Fixed the bug when limiting output records to 1 caused infinity loop | -| 0.1.6 | 2021-06-24 | [4009](https://github.com/airbytehq/airbyte/pull/4009) | Add pages, price rules and discount codes streams | -| 0.1.5 | 2021-06-10 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.4 | 2021-06-09 | [3926](https://github.com/airbytehq/airbyte/pull/3926) | New attributes to Orders schema | -| 0.1.3 | 2021-06-08 | [3787](https://github.com/airbytehq/airbyte/pull/3787) | Add Native Shopify Source Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.1.39 | 2022-10-13 | [17962](https://github.com/airbytehq/airbyte/pull/17962) | Add metafield streams; support for nested list streams | +| 0.1.38 | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777) | Fixed `404` for configured streams, fix missing `cursor` error for old records | +| 0.1.37 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.36 | 2022-03-22 | [9850](https://github.com/airbytehq/airbyte/pull/9850) | Added `BalanceTransactions` stream | +| 0.1.35 | 2022-03-07 | [10915](https://github.com/airbytehq/airbyte/pull/10915) | Fix a bug which caused `full-refresh` syncs of child REST entities configured for `incremental` | +| 0.1.34 | 2022-03-02 | [10794](https://github.com/airbytehq/airbyte/pull/10794) | Minor specification re-order, fixed links in documentation | +| 0.1.33 | 2022-02-17 | [10419](https://github.com/airbytehq/airbyte/pull/10419) | Fixed wrong field type for tax_exemptions for `Abandoned_checkouts` stream | +| 0.1.32 | 2022-02-18 | [10449](https://github.com/airbytehq/airbyte/pull/10449) | Added `tender_transactions` stream | +| 0.1.31 | 2022-02-08 | [10175](https://github.com/airbytehq/airbyte/pull/10175) | Fixed compatibility issues for legacy user config | +| 0.1.30 | 2022-01-24 | [9648](https://github.com/airbytehq/airbyte/pull/9648) | Added permission validation before sync | +| 0.1.29 | 2022-01-20 | [9049](https://github.com/airbytehq/airbyte/pull/9248) | Added `shop_url` to the record for all streams | +| 0.1.28 | 2022-01-19 | [9591](https://github.com/airbytehq/airbyte/pull/9591) | Implemented `OAuth2.0` authentication method for Airbyte Cloud | +| 0.1.27 | 2021-12-22 | [9049](https://github.com/airbytehq/airbyte/pull/9049) | Update connector fields title/description | +| 0.1.26 | 2021-12-14 | [8597](https://github.com/airbytehq/airbyte/pull/8597) | Fix `mismatched number of tables` for base-normalization, increased performance of `order_refunds` stream | +| 0.1.25 | 2021-12-02 | [8297](https://github.com/airbytehq/airbyte/pull/8297) | Added Shop stream | +| 0.1.24 | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783) | Reviewed and corrected schemas for all streams | +| 0.1.23 | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973) | Added `InventoryItems` | +| 0.1.22 | 2021-10-18 | [7101](https://github.com/airbytehq/airbyte/pull/7107) | Added FulfillmentOrders, Fulfillments streams | +| 0.1.21 | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382) | Fixed `InventoryLevels` primary key | +| 0.1.20 | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063) | Added `Location` and `InventoryLevels` as streams | +| 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option | +| 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema | +| 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers | +| 0.1.16 | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945) | Fixed the connector's performance for `Incremental refresh` | +| 0.1.15 | 2021-09-02 | [5853](https://github.com/airbytehq/airbyte/pull/5853) | Fixed `amount` type in `order_refund` schema | +| 0.1.14 | 2021-09-02 | [5801](https://github.com/airbytehq/airbyte/pull/5801) | Fixed `line_items/discount allocations` & `duties` parts of `orders` schema | +| 0.1.13 | 2021-08-17 | [5470](https://github.com/airbytehq/airbyte/pull/5470) | Fixed rate limits throttling | +| 0.1.12 | 2021-08-09 | [5276](https://github.com/airbytehq/airbyte/pull/5276) | Add status property to product schema | +| 0.1.11 | 2021-07-23 | [4943](https://github.com/airbytehq/airbyte/pull/4943) | Fix products schema up to API 2021-07 | +| 0.1.10 | 2021-07-19 | [4830](https://github.com/airbytehq/airbyte/pull/4830) | Fix for streams json schemas, upgrade to API version 2021-07 | +| 0.1.9 | 2021-07-04 | [4472](https://github.com/airbytehq/airbyte/pull/4472) | Incremental sync is now using updated\_at instead of since\_id by default | +| 0.1.8 | 2021-06-29 | [4121](https://github.com/airbytehq/airbyte/pull/4121) | Add draft orders stream | +| 0.1.7 | 2021-06-26 | [4290](https://github.com/airbytehq/airbyte/pull/4290) | Fixed the bug when limiting output records to 1 caused infinity loop | +| 0.1.6 | 2021-06-24 | [4009](https://github.com/airbytehq/airbyte/pull/4009) | Add pages, price rules and discount codes streams | +| 0.1.5 | 2021-06-10 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.4 | 2021-06-09 | [3926](https://github.com/airbytehq/airbyte/pull/3926) | New attributes to Orders schema | +| 0.1.3 | 2021-06-08 | [3787](https://github.com/airbytehq/airbyte/pull/3787) | Add Native Shopify Source Connector |