🎉 Source Shopify: support oauth flow (airbytehq#6951)

schlattk · Jan 4, 2022 · 43089e4 · 43089e4
1 parent d67fae9
commit 43089e4
Show file tree

Hide file tree

Showing 13 changed files with 142 additions and 25 deletions.
diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -147,6 +147,7 @@ jobs:
           SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }}
           SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }}
           SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }}
+          SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }}
           SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }}
           SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }}
           SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }}

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -143,6 +143,7 @@ jobs:
           SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }}
           SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }}
           SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }}
+          SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }}
           SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }}
           SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }}
           SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }}

diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
   "name": "Shopify",
   "dockerRepository": "airbyte/source-shopify",
-  "dockerImageTag": "0.1.18",
+  "dockerImageTag": "0.1.19",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -174,7 +174,7 @@
 - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
   name: Shopify
   dockerRepository: airbyte/source-shopify
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
   sourceType: api
 - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6

diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile
@@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.18
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/source-shopify
diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml
@@ -7,8 +7,13 @@ tests:
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_oauth_config.json"
+      status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+    - config_path: "secrets/config_oauth.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json
@@ -1,5 +1,8 @@
 {
   "shop": "SHOP_NAME",
   "start_date": "2020-11-01",
-  "api_password": "API_PASSWORD"
-}
+  "auth_method": {
+    "auth_method": "api_password",
+    "api_password": "SOME_API_PASSWORD"
+  }
+}
diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json
@@ -0,0 +1,10 @@
+{
+  "shop": "SHOP_NAME",
+  "start_date": "2020-11-01",
+  "auth_method": {
+    "auth_method": "access_token",
+    "client_id": "SOME_CLIENT_ID",
+    "client_secret": "SOME_CLIENT_SECRET",
+    "access_token": "SOME_ACCESS_TOKEN"
+  }
+}
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Mapping
+
+from airbyte_cdk import AirbyteLogger
+from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
+
+
+class NotImplementedAuth(Exception):
+    """ Not implemented Auth option error"""
+
+    logger = AirbyteLogger()
+
+    def __init__(self, auth_method: str = None):
+        self.message = f"Not implemented Auth method = {auth_method}"
+        super().__init__(self.logger.error(self.message))
+
+
+class ShopifyAuthenticator(TokenAuthenticator):
+
+    """
+    Making Authenticator to be able to accept Header-Based authentication.
+    """
+
+    def __init__(self, config: Mapping[str, Any]):
+        self.config = config
+
+    def get_auth_header(self) -> Mapping[str, Any]:
+
+        auth_header: str = "X-Shopify-Access-Token"
+        auth_method: Dict = self.config["auth_method"]
+        auth_option: str = auth_method.get("auth_method")
+
+        if auth_option == "access_token":
+            return {auth_header: auth_method.get("access_token")}
+        elif auth_option == "api_password":
+            return {auth_header: auth_method.get("api_password")}
+        else:
+            raise NotImplementedAuth(auth_option)
diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py
@@ -12,8 +12,8 @@
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
 
+from .auth import ShopifyAuthenticator
 from .transform import DataTypeEnforcer
 from .utils import EagerlyCachedStreamState as stream_state_cache
 from .utils import ShopifyRateLimiter as limiter
@@ -325,25 +325,14 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"price_rules/{price_rule_id}/{self.data_field}.json"
 
 
-class ShopifyAuthenticator(TokenAuthenticator):
-
-    """
-    Making Authenticator to be able to accept Header-Based authentication.
-    """
-
-    def get_auth_header(self) -> Mapping[str, Any]:
-        return {"X-Shopify-Access-Token": f"{self._token}"}
-
-
 class SourceShopify(AbstractSource):
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
 
         """
         Testing connection availability for the connector.
         """
-        auth = ShopifyAuthenticator(token=config["api_password"]).get_auth_header()
+        auth = ShopifyAuthenticator(config).get_auth_header()
         api_version = "2021-07"  # Latest Stable Release
-
         url = f"https://{config['shop']}.myshopify.com/admin/api/{api_version}/shop.json"
 
         try:
@@ -359,8 +348,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         Mapping a input config of the user input configuration as defined in the connector spec.
         Defining streams to run.
         """
-        # config["ex_state"] = super().exposed_state
-        config["authenticator"] = ShopifyAuthenticator(token=config["api_password"])
+        config["authenticator"] = ShopifyAuthenticator(config)
 
         return [
             Customers(config),

diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json
@@ -4,7 +4,7 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Shopify Source CDK Specifications",
     "type": "object",
-    "required": ["shop", "start_date", "api_password"],
+    "required": ["shop", "start_date", "auth_method"],
     "additionalProperties": false,
     "properties": {
       "shop": {
@@ -17,11 +17,68 @@
         "examples": ["2021-01-01"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
       },
-      "api_password": {
-        "type": "string",
-        "description": "The API PASSWORD for a private application in Shopify shop.",
-        "airbyte_secret": true
+      "auth_method": {
+        "title": "Shopify Authorization Method",
+        "type": "object",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["client_id","client_secret","access_token"],
+            "properties": {
+              "auth_method": {
+                "type": "string",
+                "const": "access_token",
+                "enum": ["access_token"],
+                "default": "access_token",
+                "order": 0
+              },
+              "client_id": {
+                "type": "string",
+                "description": "The API Key of the Shopify developer application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "description": "The API Secret the Shopify developer application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "description": "Access Token for making authenticated requests.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "API Password",
+            "type": "object",
+            "required": ["api_password"],
+            "properties": {
+              "auth_method": {
+                "type": "string",
+                "const": "api_password",
+                "enum": ["api_password"],
+                "default": "api_password",
+                "order": 0
+              },
+              "api_password": {
+                "type": "string",
+                "description": "The API PASSWORD for your private application in `Shopify` shop.",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       }
+    } 
+  },
+  "authSpecification": {
+    "auth_type": "oauth2.0",
+    "oauth2Specification": {
+      "rootObject": ["auth_method", 0],
+      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
+      "oauthFlowOutputParameters": [["access_token"]]
     }
   }
 }
diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md
@@ -74,6 +74,9 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 
 ## Getting started
 
+This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applications) athentication methods.
+
+### Connect using `API PASSWORD` option:
 1. Go to `https://YOURSTORE.myshopify.com/admin/apps/private`
 2. Enable private development if it isn't enabled.
 3. Create a private application.
@@ -82,10 +85,17 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 5. The password under the `Admin API` section is what you'll use as the `api_password` for the integration.
 6. You're ready to set up Shopify in Airbyte!
 
+### Connect using `OAuth 2.0` option:
+1. Select `OAuth 2.0` in `Shopify Authorization Method`
+2. Click on `authenticate`
+2. Proceed the authentication using your credentials for your Shopify account.
+
+
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option |
 | 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema |
 | 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers |
 | 0.1.16 | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945) | Fixed the connector's performance for `Incremental refresh` |

diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh
@@ -123,6 +123,7 @@ write_standard_creds source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS
 write_standard_creds source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS"
 write_standard_creds source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS"
 write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS"
+write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json"
 write_standard_creds source-shortio "$SOURCE_SHORTIO_TEST_CREDS"
 write_standard_creds source-slack "$SOURCE_SLACK_TEST_CREDS"
 write_standard_creds source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json"