From de803ecf22b83b2d1cf50e934b356bcd9a856980 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sun, 14 Nov 2021 19:45:09 +0200 Subject: [PATCH 01/13] add OAuth 2.0 for source-harvest Signed-off-by: Sergey Chvalyuk --- .../fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-harvest/Dockerfile | 2 +- .../source-harvest/source_harvest/auth.py | 16 +++- .../source-harvest/source_harvest/source.py | 19 ++++- .../source-harvest/source_harvest/spec.json | 37 ++++++++- .../oauth/OAuthImplementationFactory.java | 2 + .../airbyte/oauth/flows/HarvestOAuthFlow.java | 78 +++++++++++++++++++ 8 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json index 3774938b28d5..979da9076f57 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "fe2b4084-3386-4d3b-9ad6-308f61a6f1e6", "name": "Harvest", "dockerRepository": "airbyte/source-harvest", - "dockerImageTag": "0.1.5", + "dockerImageTag": "0.1.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/harvest" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a129bba53bd4..9799ca99500b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -236,7 +236,7 @@ - name: Harvest sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 dockerRepository: airbyte/source-harvest - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/harvest sourceType: api - name: HubSpot diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile index 7d16c04752ac..0b9cd8c09544 100644 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py index b7c176ddc291..4385ac568d7a 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py @@ -4,14 +4,22 @@ from typing import Any, Mapping -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator -class HarvestTokenAuthenticator(TokenAuthenticator): - def __init__(self, token: str, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): - super().__init__(token, **kwargs) +class HarvestMixin: + def __init__(self, *, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): + super().__init__(**kwargs) self.account_id = account_id self.account_id_header = account_id_header def get_auth_header(self) -> Mapping[str, Any]: return {**super().get_auth_header(), self.account_id_header: self.account_id} + + +class HarvestTokenAuthenticator(HarvestMixin, TokenAuthenticator): + pass + + +class HarvestOauth2Authenticator(HarvestMixin, Oauth2Authenticator): + pass diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py index 730e0a781a8f..eabd22590617 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py @@ -45,13 +45,26 @@ Users, ) -from .auth import HarvestTokenAuthenticator +from .auth import HarvestOauth2Authenticator, HarvestTokenAuthenticator class SourceHarvest(AbstractSource): + @staticmethod + def get_authenticator(config): + credentials = config.get("credentials") + if credentials: + return HarvestOauth2Authenticator( + token_refresh_endpoint="https://id.getharvest.com/api/v2/oauth2/token", + client_id=credentials.get("client_id"), + client_secret=credentials.get("client_secret"), + refresh_token=credentials.get("refresh_token"), + account_id=config["account_id"], + ) + return HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: - auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + auth = self.get_authenticator(config) replication_start_date = pendulum.parse(config["replication_start_date"]) users_gen = Users(authenticator=auth, replication_start_date=replication_start_date).read_records( sync_mode=SyncMode.full_refresh @@ -65,7 +78,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + auth = self.get_authenticator(config) replication_start_date = pendulum.parse(config["replication_start_date"]) from_date = replication_start_date.date() diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index 82812a0e4773..cb3993739eeb 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Harvest Spec", "type": "object", - "required": ["api_token", "account_id", "replication_start_date"], + "required": ["account_id", "replication_start_date"], "additionalProperties": false, "properties": { "api_token": { @@ -25,9 +25,42 @@ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "type": "string" + }, + "credentials": { + "type": "object", + "title": "Authenticate via Harvest (OAuth 2.0)", + "required": ["client_id", "client_secret", "refresh_token"], + "additionalProperties": false, + "properties": { + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your application" + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The client secret of your application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "A refresh token generated using the above client ID and secret", + "airbyte_secret": true + } + } } } }, "supportsIncremental": true, - "supported_destination_sync_modes": ["append"] + "supported_destination_sync_modes": ["append"], + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials"], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } + } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 83a044b2ed66..e8c1f2ed0db0 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -10,6 +10,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.flows.AsanaOAuthFlow; import io.airbyte.oauth.flows.GithubOAuthFlow; +import io.airbyte.oauth.flows.HarvestOAuthFlow; import io.airbyte.oauth.flows.HubspotOAuthFlow; import io.airbyte.oauth.flows.IntercomOAuthFlow; import io.airbyte.oauth.flows.QuickbooksOAuthFlow; @@ -42,6 +43,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository, httpClient)) .put("airbyte/source-google-sheets", new GoogleSheetsOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-harvest", new HarvestOAuthFlow(configRepository, httpClient)) .put("airbyte/source-hubspot", new HubspotOAuthFlow(configRepository, httpClient)) .put("airbyte/source-intercom", new IntercomOAuthFlow(configRepository, httpClient)) .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java new file mode 100644 index 000000000000..176970133191 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class HarvestOAuthFlow extends BaseOAuth2Flow { + + private static final String AUTHORIZE_URL = "https://id.getharvest.com/oauth2/authorize"; + private static final String ACCESS_TOKEN_URL = "https://id.getharvest.com/api/v2/oauth2/token"; + + public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + /** + * Depending on the OAuth flow implementation, the URL to grant user's consent may differ, + * especially in the query parameters to be provided. This function should generate such consent URL + * accordingly. + * + * @param definitionId The configured definition ID of this client + * @param clientId The configured client ID + * @param redirectUrl the redirect URL + */ + @Override + protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl) throws IOException { + try { + return new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("response_type", "code") + .addParameter("redirect_uri", redirectUrl) + .addParameter("state", getState()) + .build().toString(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl) { + return ImmutableMap.builder() + // required + .put("client_id", clientId) + .put("redirect_uri", redirectUrl) + .put("client_secret", clientSecret) + .put("code", authCode) + .put("grant_type", "authorization_code") + .build(); + } + + /** + * Returns the URL where to retrieve the access token from. + * + */ + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + +} From 524dfd183e4fce0b1d31108f8f7fed917f644c5c Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sun, 14 Nov 2021 19:48:49 +0200 Subject: [PATCH 02/13] changelog added Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/harvest.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index b4bc7f88642c..1e9f59c2511e 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -69,6 +69,7 @@ See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/ | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | | 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields | | 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas | | 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream | From 4f84a8c41a6ac0d50128f994b9c1cb868fe59379 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Mon, 15 Nov 2021 23:00:33 +0200 Subject: [PATCH 03/13] oneOf added for spec.json Signed-off-by: Sergey Chvalyuk --- .../source-harvest/source_harvest/source.py | 10 ++- .../source-harvest/source_harvest/spec.json | 71 +++++++++++-------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py index eabd22590617..a2b60b6e2aa3 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py @@ -51,8 +51,8 @@ class SourceHarvest(AbstractSource): @staticmethod def get_authenticator(config): - credentials = config.get("credentials") - if credentials: + credentials = config.get("credentials", {}) + if credentials and "client_id" in credentials: return HarvestOauth2Authenticator( token_refresh_endpoint="https://id.getharvest.com/api/v2/oauth2/token", client_id=credentials.get("client_id"), @@ -60,7 +60,11 @@ def get_authenticator(config): refresh_token=credentials.get("refresh_token"), account_id=config["account_id"], ) - return HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + + api_token = credentials.get("api_token", config.get("api_token")) + if not api_token: + raise Exception("Config validation error: 'api_token' is a required property") + return HarvestTokenAuthenticator(token=api_token, account_id=config["account_id"]) def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index cb3993739eeb..61b7eb8d4a7c 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -5,14 +5,8 @@ "title": "Harvest Spec", "type": "object", "required": ["account_id", "replication_start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { - "api_token": { - "title": "API Token", - "description": "Harvest API Token.", - "airbyte_secret": true, - "type": "string" - }, "account_id": { "title": "Account ID", "description": "Harvest account ID. Required for all Harvest requests in pair with API Key", @@ -27,29 +21,50 @@ "type": "string" }, "credentials": { + "title": "Authentication mechanism", + "description": "Choose how to authenticate to Harvest", "type": "object", - "title": "Authenticate via Harvest (OAuth 2.0)", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": false, - "properties": { - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your application" - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The client secret of your application", - "airbyte_secret": true + "oneOf": [ + { + "type": "object", + "title": "Authenticate via Harvest (Oauth)", + "required": ["client_id", "client_secret", "refresh_token"], + "additionalProperties": false, + "properties": { + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your application" + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The client secret of your application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "A refresh token generated using the above client ID and secret", + "airbyte_secret": true + } + } }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "A refresh token generated using the above client ID and secret", - "airbyte_secret": true + { + "type": "object", + "title": "Authenticate with Personal Access Token", + "required": ["api_token"], + "additionalProperties": false, + "properties": { + "api_token": { + "title": "Personal Access Token", + "description": "Log into Harvest and then create new personal access token.", + "type": "string", + "airbyte_secret": true + } + } } - } + ] } } }, @@ -58,7 +73,7 @@ "authSpecification": { "auth_type": "oauth2.0", "oauth2Specification": { - "rootObject": ["credentials"], + "rootObject": ["credentials", 0], "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["refresh_token"]] } From 957301549df7a704b3b4a003b78991f0489292df Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Mon, 15 Nov 2021 23:33:57 +0200 Subject: [PATCH 04/13] old_config.json, config_oauth.json added Signed-off-by: Sergey Chvalyuk --- .../source-harvest/acceptance-test-config.yml | 4 ++++ .../source-harvest/source_harvest/spec.json | 14 ++++++++++++++ tools/bin/ci_credentials.sh | 2 ++ 3 files changed, 20 insertions(+) diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml index 5632dd24e979..b23a0698ff48 100644 --- a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml @@ -5,6 +5,10 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index 61b7eb8d4a7c..9a68864150d9 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -31,6 +31,13 @@ "required": ["client_id", "client_secret", "refresh_token"], "additionalProperties": false, "properties": { + "auth_type": { + "type": "string", + "const": "Client", + "enum": ["Client"], + "default": "Client", + "order": 0 + }, "client_id": { "title": "Client ID", "type": "string", @@ -56,6 +63,13 @@ "required": ["api_token"], "additionalProperties": false, "properties": { + "auth_type": { + "type": "string", + "const": "Token", + "enum": ["Token"], + "default": "Token", + "order": 0 + }, "api_token": { "title": "Personal Access Token", "description": "Log into Harvest and then create new personal access token.", diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index f08c7081be92..6832551f2b49 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -225,6 +225,8 @@ read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPO read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" +read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" +read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS_OLD" "old_config.json" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" From 62b6972d1d8758d7a2e9a53e53f3fd5c3459b9d0 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 16 Nov 2021 16:28:48 +0200 Subject: [PATCH 05/13] fix read_secret names HARVEST_* -> SECRET_*_HARVEST Signed-off-by: Sergey Chvalyuk --- tools/bin/ci_credentials.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index d727f4d0c121..f1e2d367a8e6 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -224,9 +224,9 @@ read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.j read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" -read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" -read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" -read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS_OLD" "old_config.json" +read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS" +read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS_OAUTH" "config_oauth.json" +read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS_OLD" "old_config.json" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" From 4ad0e185062ed2a9817a3f0f42179c4547b6f835 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 16 Nov 2021 17:16:41 +0200 Subject: [PATCH 06/13] bugfix: remove SECRET_*_HARVEST we don't need for GSM Signed-off-by: Sergey Chvalyuk --- tools/bin/ci_credentials.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index f1e2d367a8e6..9ee0b7b1ab41 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -224,9 +224,6 @@ read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.j read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" -read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS" -read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS_OAUTH" "config_oauth.json" -read_secrets source-harvest "$SECRET_SOURCE_HARVEST_CREDS_OLD" "old_config.json" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" From a570dc1de57c7b8845878010f92828f513df992f Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 16 Nov 2021 18:47:40 +0200 Subject: [PATCH 07/13] added auth class docs Signed-off-by: Sergey Chvalyuk --- .../source-harvest/source_harvest/auth.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py index 4385ac568d7a..fd52d0abe129 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py @@ -8,6 +8,10 @@ class HarvestMixin: + """ + Mixin class for providing additional HTTP header for specifying account ID + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/ + """ def __init__(self, *, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): super().__init__(**kwargs) self.account_id = account_id @@ -18,8 +22,14 @@ def get_auth_header(self) -> Mapping[str, Any]: class HarvestTokenAuthenticator(HarvestMixin, TokenAuthenticator): - pass + """ + Auth class for Personal Access Token + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens + """ class HarvestOauth2Authenticator(HarvestMixin, Oauth2Authenticator): - pass + """ + Auth class for OAuth2 + https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#for-server-side-applications + """ From 003efc95ad46dfdcf1189281efb24844add72336 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 16 Nov 2021 20:49:27 +0200 Subject: [PATCH 08/13] HarvestOAuthFlowTest.java added Signed-off-by: Sergey Chvalyuk --- .../oauth/flows/HarvestOAuthFlowTest.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java new file mode 100644 index 000000000000..299a573c03bf --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class HarvestOAuthFlowTest { + + private UUID workspaceId; + private UUID definitionId; + private HarvestOAuthFlow harvestOAuthFlow; + private HttpClient httpClient; + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private static String getConstantState() { + return "state"; + } + + @BeforeEach + public void setup() throws IOException, JsonValidationException { + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + ConfigRepository configRepository = mock(ConfigRepository.class); + httpClient = mock(HttpClient.class); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()))))); + harvestOAuthFlow = new HarvestOAuthFlow(configRepository, httpClient, HarvestOAuthFlowTest::getConstantState); + + } + + @Test + public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { + final String concentUrl = + harvestOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals(concentUrl, + "https://id.getharvest.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"); + } + + @Test + public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { + + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + harvestOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + Map credentials = (Map) actualQueryParams.get("credentials"); + assertEquals(credentials.get("refresh_token"), "refresh_token_response"); + } + +} From 14964a2abeff07c3fd42a91a45b06694f3299e46 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 18 Nov 2021 00:52:36 +0200 Subject: [PATCH 09/13] fix formatConsentUrl signature, fix oauth test Signed-off-by: Sergey Chvalyuk --- .../airbyte/oauth/flows/HarvestOAuthFlow.java | 7 +- .../oauth/flows/HarvestOAuthFlowTest.java | 76 ++----------------- 2 files changed, 14 insertions(+), 69 deletions(-) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java index 176970133191..d7cceff0de02 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java @@ -4,6 +4,7 @@ package io.airbyte.oauth.flows; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.BaseOAuth2Flow; @@ -38,7 +39,11 @@ public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClien * @param redirectUrl the redirect URL */ @Override - protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl) throws IOException { + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { try { return new URIBuilder(AUTHORIZE_URL) .addParameter("client_id", clientId) diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java index 299a573c03bf..96bf7a75caa9 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java @@ -4,78 +4,18 @@ package io.airbyte.oauth.flows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import io.airbyte.oauth.BaseOAuthFlow; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.validation.json.JsonValidationException; -import java.io.IOException; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +public class HarvestOAuthFlowTest extends BaseOAuthFlowTest { -public class HarvestOAuthFlowTest { - - private UUID workspaceId; - private UUID definitionId; - private HarvestOAuthFlow harvestOAuthFlow; - private HttpClient httpClient; - - private static final String REDIRECT_URL = "https://airbyte.io"; - - private static String getConstantState() { - return "state"; - } - - @BeforeEach - public void setup() throws IOException, JsonValidationException { - workspaceId = UUID.randomUUID(); - definitionId = UUID.randomUUID(); - ConfigRepository configRepository = mock(ConfigRepository.class); - httpClient = mock(HttpClient.class); - when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withSourceDefinitionId(definitionId) - .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() - .put("client_id", "test_client_id") - .put("client_secret", "test_client_secret") - .build()))))); - harvestOAuthFlow = new HarvestOAuthFlow(configRepository, httpClient, HarvestOAuthFlowTest::getConstantState); - - } - - @Test - public void testGetSourceConcentUrl() throws IOException, ConfigNotFoundException { - final String concentUrl = - harvestOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); - assertEquals(concentUrl, - "https://id.getharvest.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"); + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new HarvestOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); } - @Test - public void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigNotFoundException { - - Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); - final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); - when(httpClient.send(any(), any())).thenReturn(response); - - final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = - harvestOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - Map credentials = (Map) actualQueryParams.get("credentials"); - assertEquals(credentials.get("refresh_token"), "refresh_token_response"); + @Override + protected String getExpectedConsentUrl() { + return "https://id.getharvest.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"; } } From 1383bbe086ceecd27a7f068067d6735b0c3be966 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 19 Nov 2021 15:56:34 +0200 Subject: [PATCH 10/13] advancedAuth added Signed-off-by: Sergey Chvalyuk --- .../source-harvest/source_harvest/spec.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index 9a68864150d9..c9c29d2c75b6 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -91,5 +91,34 @@ "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["refresh_token"]] } + }, + "advancedAuth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", 0, "refresh_token"] + } + }, + "complete_oauth_server_input_specification": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + }, + "complete_oauth_server_output_specification": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", 0, "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", 0, "client_secret"] + } + } + } } } From ac50b02e228ada30b2f4a09b674a13256335cbc4 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 19 Nov 2021 18:12:59 +0200 Subject: [PATCH 11/13] advancedAuth fixed Signed-off-by: Sergey Chvalyuk --- .../source-harvest/source_harvest/spec.json | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index c9c29d2c75b6..80f07555dc36 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -94,29 +94,43 @@ }, "advancedAuth": { "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", "oauth_config_specification": { "complete_oauth_output_specification": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", 0, "refresh_token"] + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } } }, "complete_oauth_server_input_specification": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } } }, "complete_oauth_server_output_specification": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", 0, "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", 0, "client_secret"] + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } } } } From 3b4b4d56fe82ed16c09def66485115f179ca002a Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 19 Nov 2021 20:04:12 +0200 Subject: [PATCH 12/13] ordering added Signed-off-by: Sergey Chvalyuk --- .../connectors/source-harvest/source_harvest/spec.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json index 80f07555dc36..17269a455dd5 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -11,19 +11,22 @@ "title": "Account ID", "description": "Harvest account ID. Required for all Harvest requests in pair with API Key", "airbyte_secret": true, - "type": "string" + "type": "string", + "order": 0 }, "replication_start_date": { "title": "Replication Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "type": "string" + "type": "string", + "order": 1 }, "credentials": { "title": "Authentication mechanism", "description": "Choose how to authenticate to Harvest", "type": "object", + "order": 2, "oneOf": [ { "type": "object", From 202013aafad4a9d1ce2fd16976ed04c02b74377a Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 19 Nov 2021 20:21:43 +0200 Subject: [PATCH 13/13] source_specs.yaml updated Signed-off-by: Sergey Chvalyuk --- .../src/main/resources/seed/source_specs.yaml | 116 ++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index bef52af9233b..9b47693dd073 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2321,7 +2321,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-harvest:0.1.5" +- dockerImage: "airbyte/source-harvest:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/harvest" connectionSpecification: @@ -2329,22 +2329,17 @@ title: "Harvest Spec" type: "object" required: - - "api_token" - "account_id" - "replication_start_date" - additionalProperties: false + additionalProperties: true properties: - api_token: - title: "API Token" - description: "Harvest API Token." - airbyte_secret: true - type: "string" account_id: title: "Account ID" description: "Harvest account ID. Required for all Harvest requests in pair\ \ with API Key" airbyte_secret: true type: "string" + order: 0 replication_start_date: title: "Replication Start Date" description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ @@ -2353,11 +2348,116 @@ examples: - "2017-01-25T00:00:00Z" type: "string" + order: 1 + credentials: + title: "Authentication mechanism" + description: "Choose how to authenticate to Harvest" + type: "object" + order: 2 + oneOf: + - type: "object" + title: "Authenticate via Harvest (Oauth)" + required: + - "client_id" + - "client_secret" + - "refresh_token" + additionalProperties: false + properties: + auth_type: + type: "string" + const: "Client" + enum: + - "Client" + default: "Client" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Client ID of your application" + client_secret: + title: "Client Secret" + type: "string" + description: "The client secret of your application" + airbyte_secret: true + refresh_token: + title: "Refresh Token" + type: "string" + description: "A refresh token generated using the above client ID\ + \ and secret" + airbyte_secret: true + - type: "object" + title: "Authenticate with Personal Access Token" + required: + - "api_token" + additionalProperties: false + properties: + auth_type: + type: "string" + const: "Token" + enum: + - "Token" + default: "Token" + order: 0 + api_token: + title: "Personal Access Token" + description: "Log into Harvest and then create new personal access token." + type: "string" + airbyte_secret: true supportsIncremental: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: - "append" + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" + advancedAuth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "Client" + oauth_config_specification: + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "credentials" + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-hubspot:0.1.24" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"