Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎉 Source Amazon Ads: Implement OAuth2.0 #11430

Merged
merged 17 commits into from
Apr 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
- name: Amazon Ads
sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
dockerRepository: airbyte/source-amazon-ads
dockerImageTag: 0.1.4
dockerImageTag: 0.1.5
documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
icon: amazonads.svg
sourceType: api
Expand Down
51 changes: 41 additions & 10 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,18 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-amazon-ads:0.1.4"
- dockerImage: "airbyte/source-amazon-ads:0.1.5"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-ads"
connectionSpecification:
title: "Amazon Ads Spec"
type: "object"
properties:
auth_type:
title: "Auth Type"
const: "oauth2.0"
order: 0
type: "string"
client_id:
title: "Client Id"
description: "Oauth client id <a href=\"https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app\"\
Expand All @@ -107,15 +112,6 @@
name: "Client secret"
airbyte_secret: true
type: "string"
scope:
title: "Scope"
description: "By default its advertising::campaign_management, but customers\
\ may need to set scope to cpc_advertising:campaign_management."
default: "advertising::campaign_management"
name: "Client scope"
examples:
- "cpc_advertising:campaign_management"
type: "string"
refresh_token:
title: "Refresh Token"
description: "Oauth 2.0 refresh_token, <a href=\"https://developer.amazon.com/docs/login-with-amazon/conceptual-overview.html\"\
Expand Down Expand Up @@ -174,9 +170,44 @@
- "client_id"
- "client_secret"
- "refresh_token"
additionalProperties: true
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
advanced_auth:
auth_flow_type: "oauth2.0"
predicate_key:
- "auth_type"
predicate_value: "oauth2.0"
oauth_config_specification:
complete_oauth_output_specification:
type: "object"
additionalProperties: false
properties:
refresh_token:
type: "string"
path_in_connector_config:
- "refresh_token"
complete_oauth_server_input_specification:
type: "object"
additionalProperties: false
properties:
client_id:
type: "string"
client_secret:
type: "string"
complete_oauth_server_output_specification:
type: "object"
additionalProperties: false
properties:
client_id:
type: "string"
path_in_connector_config:
- "client_id"
client_secret:
type: "string"
path_in_connector_config:
- "client_secret"
- dockerImage: "airbyte/source-amazon-seller-partner:0.2.15"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.name=airbyte/source-amazon-ads
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@
"connectionSpecification": {
"title": "Amazon Ads Spec",
"type": "object",
"additionalProperties": true,
"properties": {
"auth_type": {
"title": "Auth Type",
"const": "oauth2.0",
"order": 0,
"type": "string"
},
"client_id": {
"title": "Client Id",
"description": "Oauth client id <a href=\"https://advertising.amazon.com/API/docs/en-us/setting-up/step-1-create-lwa-app\">How to create your Login with Amazon</a>",
Expand All @@ -17,14 +24,6 @@
"airbyte_secret": true,
"type": "string"
},
"scope": {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why was scope removed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to amazon auth docs we need only scope to get refresh_token.
and we don't need scope to exchange refresh_token -> access_token.

It means scope is useless here because we already have refresh_token as connector input parameter.

I this PR I moved scope from python side to Java side
because we need scope only on stage where we get refresh_token

"title": "Scope",
"description": "By default its advertising::campaign_management, but customers may need to set scope to cpc_advertising:campaign_management.",
"default": "advertising::campaign_management",
"name": "Client scope",
"examples": ["cpc_advertising:campaign_management"],
"type": "string"
},
"refresh_token": {
"title": "Refresh Token",
"description": "Oauth 2.0 refresh_token, <a href=\"https://developer.amazon.com/docs/login-with-amazon/conceptual-overview.html\">read details here</a>",
Expand Down Expand Up @@ -74,5 +73,56 @@
}
},
"required": ["client_id", "client_secret", "refresh_token"]
},
"advanced_auth": {
"auth_flow_type": "oauth2.0",
"predicate_key": [
"auth_type"
],
"predicate_value": "oauth2.0",
"oauth_config_specification": {
"complete_oauth_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"refresh_token": {
"type": "string",
"path_in_connector_config": [
"refresh_token"
]
}
}
},
"complete_oauth_server_input_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"client_id": {
"type": "string"
},
"client_secret": {
"type": "string"
}
}
},
"complete_oauth_server_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"client_id": {
"type": "string",
"path_in_connector_config": [
"client_id"
]
},
"client_secret": {
"type": "string",
"path_in_connector_config": [
"client_secret"
]
}
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator

from .schemas import Profile
from .spec import AmazonAdsConfig
from .spec import AmazonAdsConfig, advanced_auth
from .streams import (
Profiles,
SponsoredBrandsAdGroups,
Expand Down Expand Up @@ -95,6 +95,7 @@ def spec(self, *args) -> ConnectorSpecification:
return ConnectorSpecification(
documentationUrl="https://docs.airbyte.io/integrations/sources/amazon-ads",
connectionSpecification=AmazonAdsConfig.schema(),
advanced_auth=advanced_auth,
)

@staticmethod
Expand All @@ -104,7 +105,6 @@ def _make_authenticator(config: AmazonAdsConfig):
client_id=config.client_id,
client_secret=config.client_secret,
refresh_token=config.refresh_token,
scopes=[config.scope],
)

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,21 @@

from typing import List

from pydantic import BaseModel, Field
from airbyte_cdk.models import AdvancedAuth, AuthFlowType, OAuthConfigSpecification
from pydantic import BaseModel, Extra, Field
from source_amazon_ads.constants import AmazonAdsRegion


class AmazonAdsConfig(BaseModel):
class Config:
title = "Amazon Ads Spec"
# ignore extra attributes during model initialization
# https://pydantic-docs.helpmanual.io/usage/model_config/
extra = Extra.ignore
# it's default, but better to be more explicit
schema_extra = {"additionalProperties": True}

auth_type: str = Field(default="oauth2.0", const=True, order=0)

client_id: str = Field(
name="Client ID",
Expand All @@ -28,19 +36,6 @@ class Config:
airbyte_secret=True,
)

# Amazon docs don't describe which of the below scopes to use under what circumstances so
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make sure additionalproperties=true so existing configs don't break

# we default to the first but allow the user to override it
scope: str = Field(
"advertising::campaign_management",
name="Client scope",
examples=[
"cpc_advertising:campaign_management",
],
description=(
"By default its advertising::campaign_management,"
" but customers may need to set scope to cpc_advertising:campaign_management."
),
)
refresh_token: str = Field(
name="Oauth refresh token",
description=(
Expand Down Expand Up @@ -80,16 +75,38 @@ class Config:
)

@classmethod
def schema(cls, **kvargs):
schema = super().schema(**kvargs)
# We are using internal _host parameter to set API host to sandbox
# environment for SAT but dont want it to be visible for end users,
# filter out it from the jsonschema output
schema["properties"] = {name: desc for name, desc in schema["properties"].items() if not name.startswith("_")}
def schema(cls, **kwargs):
schema = super().schema(**kwargs)
# Transform pydantic generated enum for region
definitions = schema.pop("definitions", None)
if definitions:
schema["properties"]["region"].update(definitions["AmazonAdsRegion"])
schema["properties"]["region"].pop("allOf", None)
schema["properties"]["region"].pop("$ref", None)
return schema


advanced_auth = AdvancedAuth(
auth_flow_type=AuthFlowType.oauth2_0,
predicate_key=["auth_type"],
predicate_value="oauth2.0",
oauth_config_specification=OAuthConfigSpecification(
complete_oauth_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["refresh_token"]}},
},
complete_oauth_server_input_specification={
"type": "object",
"additionalProperties": False,
"properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}},
},
complete_oauth_server_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {
"client_id": {"type": "string", "path_in_connector_config": ["client_id"]},
"client_secret": {"type": "string", "path_in_connector_config": ["client_secret"]},
},
},
),
)
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class OAuthImplementationFactory {

public OAuthImplementationFactory(final ConfigRepository configRepository, final HttpClient httpClient) {
OAUTH_FLOW_MAPPING = ImmutableMap.<String, OAuthFlowImplementation>builder()
.put("airbyte/source-amazon-ads", new AmazonAdsOAuthFlow(configRepository, httpClient))
.put("airbyte/source-asana", new AsanaOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-pages", new FacebookPagesOAuthFlow(configRepository, httpClient))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.oauth.flows;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.oauth.BaseOAuth2Flow;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.apache.http.client.utils.URIBuilder;

public class AmazonAdsOAuthFlow extends BaseOAuth2Flow {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you add a unit test for this please?


private static final String AUTHORIZE_URL = "https://www.amazon.com/ap/oa";
private static final String ACCESS_TOKEN_URL = "https://api.amazon.com/auth/o2/token";

public AmazonAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
super(configRepository, httpClient);
}

public AmazonAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
super(configRepository, httpClient, stateSupplier);
}

/**
* Depending on the OAuth flow implementation, the URL to grant user's consent may differ,
* especially in the query parameters to be provided. This function should generate such consent URL
* accordingly.
*
* @param definitionId The configured definition ID of this client
* @param clientId The configured client ID
* @param redirectUrl the redirect URL
*/
@Override
protected String formatConsentUrl(final UUID definitionId,
final String clientId,
final String redirectUrl,
final JsonNode inputOAuthConfiguration)
throws IOException {
try {
return new URIBuilder(AUTHORIZE_URL)
.addParameter("client_id", clientId)
.addParameter("scope", "advertising::campaign_management")
.addParameter("response_type", "code")
.addParameter("redirect_uri", redirectUrl)
.addParameter("state", getState())
.build().toString();
} catch (final URISyntaxException e) {
throw new IOException("Failed to format Consent URL for OAuth flow", e);
}
}

@Override
protected Map<String, String> getAccessTokenQueryParameters(final String clientId,
final String clientSecret,
final String authCode,
final String redirectUrl) {
return ImmutableMap.<String, String>builder()
// required
.put("client_id", clientId)
.put("redirect_uri", redirectUrl)
.put("client_secret", clientSecret)
.put("code", authCode)
.put("grant_type", "authorization_code")
.build();
}

/**
* Returns the URL where to retrieve the access token from.
*
*/
@Override
protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
return ACCESS_TOKEN_URL;
}

}
Loading