🎉 Source Harvest: add OAuth 2.0

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "fe2b4084-3386-4d3b-9ad6-308f61a6f1e6",
   "name": "Harvest",
   "dockerRepository": "airbyte/source-harvest",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/harvest"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -236,7 +236,7 @@
 - name: Harvest
   sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6
   dockerRepository: airbyte/source-harvest
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/harvest
   sourceType: api
 - name: HubSpot

airbyte-integrations/connectors/source-harvest/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-harvest

airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml

@@ -5,6 +5,10 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/old_config.json"
+      status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:

airbyte-integrations/connectors/source-harvest/source_harvest/auth.py

@@ -4,14 +4,32 @@
 
 from typing import Any, Mapping
 
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator
 
 
-class HarvestTokenAuthenticator(TokenAuthenticator):
-    def __init__(self, token: str, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs):
-        super().__init__(token, **kwargs)
+class HarvestMixin:
+    """
+    Mixin class for providing additional HTTP header for specifying account ID
+    https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/
+    """
+    def __init__(self, *, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs):
+        super().__init__(**kwargs)
         self.account_id = account_id
         self.account_id_header = account_id_header
 
     def get_auth_header(self) -> Mapping[str, Any]:
         return {**super().get_auth_header(), self.account_id_header: self.account_id}
+
+
+class HarvestTokenAuthenticator(HarvestMixin, TokenAuthenticator):
+    """
+    Auth class for Personal Access Token
+    https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens
+    """
+
+
+class HarvestOauth2Authenticator(HarvestMixin, Oauth2Authenticator):
+    """
+    Auth class for OAuth2
+    https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#for-server-side-applications
+    """

airbyte-integrations/connectors/source-harvest/source_harvest/source.py

@@ -45,13 +45,30 @@
     Users,
 )
 
-from .auth import HarvestTokenAuthenticator
+from .auth import HarvestOauth2Authenticator, HarvestTokenAuthenticator
 
 
 class SourceHarvest(AbstractSource):
+    @staticmethod
+    def get_authenticator(config):
+        credentials = config.get("credentials", {})
+        if credentials and "client_id" in credentials:
+            return HarvestOauth2Authenticator(
+                token_refresh_endpoint="https://id.getharvest.com/api/v2/oauth2/token",
+                client_id=credentials.get("client_id"),
+                client_secret=credentials.get("client_secret"),
+                refresh_token=credentials.get("refresh_token"),
+                account_id=config["account_id"],
+            )
+
+        api_token = credentials.get("api_token", config.get("api_token"))
+        if not api_token:
+            raise Exception("Config validation error: 'api_token' is a required property")
+        return HarvestTokenAuthenticator(token=api_token, account_id=config["account_id"])
+
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
         try:
-            auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"])
+            auth = self.get_authenticator(config)
             replication_start_date = pendulum.parse(config["replication_start_date"])
             users_gen = Users(authenticator=auth, replication_start_date=replication_start_date).read_records(
                 sync_mode=SyncMode.full_refresh
@@ -65,7 +82,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         """
         :param config: A Mapping of the user input configuration as defined in the connector spec.
         """
-        auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"])
+        auth = self.get_authenticator(config)
         replication_start_date = pendulum.parse(config["replication_start_date"])
         from_date = replication_start_date.date()
 

airbyte-integrations/connectors/source-harvest/source_harvest/spec.json

@@ -4,15 +4,9 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Harvest Spec",
     "type": "object",
-    "required": ["api_token", "account_id", "replication_start_date"],
-    "additionalProperties": false,
+    "required": ["account_id", "replication_start_date"],
+    "additionalProperties": true,
     "properties": {
-      "api_token": {
-        "title": "API Token",
-        "description": "Harvest API Token.",
-        "airbyte_secret": true,
-        "type": "string"
-      },
       "account_id": {
         "title": "Account ID",
         "description": "Harvest account ID. Required for all Harvest requests in pair with API Key",
@@ -25,9 +19,106 @@
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "examples": ["2017-01-25T00:00:00Z"],
         "type": "string"
+      },
+      "credentials": {
+        "title": "Authentication mechanism",
+        "description": "Choose how to authenticate to Harvest",
+        "type": "object",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "Authenticate via Harvest (Oauth)",
+            "required": ["client_id", "client_secret", "refresh_token"],
+            "additionalProperties": false,
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "Client",
+                "enum": ["Client"],
+                "default": "Client",
+                "order": 0
+              },
+              "client_id": {
+                "title": "Client ID",
+                "type": "string",
+                "description": "The Client ID of your application"
+              },
+              "client_secret": {
+                "title": "Client Secret",
+                "type": "string",
+                "description": "The client secret of your application",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "title": "Refresh Token",
+                "type": "string",
+                "description": "A refresh token generated using the above client ID and secret",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "type": "object",
+            "title": "Authenticate with Personal Access Token",
+            "required": ["api_token"],
+            "additionalProperties": false,
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "Token",
+                "enum": ["Token"],
+                "default": "Token",
+                "order": 0
+              },
+              "api_token": {
+                "title": "Personal Access Token",
+                "description": "Log into Harvest and then create new <a href=\"https://id.getharvest.com/developers\"> personal access token</a>.",
+                "type": "string",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       }
     }
   },
   "supportsIncremental": true,
-  "supported_destination_sync_modes": ["append"]
+  "supported_destination_sync_modes": ["append"],
+  "authSpecification": {
+    "auth_type": "oauth2.0",
+    "oauth2Specification": {
+      "rootObject": ["credentials", 0],
+      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
+      "oauthFlowOutputParameters": [["refresh_token"]]
+    }
+  },
+  "advancedAuth": {
+    "auth_flow_type": "oauth2.0",
+    "oauth_config_specification": {
+      "complete_oauth_output_specification": {
+        "refresh_token": {
+          "type": "string",
+          "path_in_connector_config": ["credentials", 0, "refresh_token"]
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "client_id": {
+          "type": "string"
+        },
+        "client_secret": {
+          "type": "string"
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "client_id": {
+          "type": "string",
+          "path_in_connector_config": ["credentials", 0, "client_id"]
+        },
+        "client_secret": {
+          "type": "string",
+          "path_in_connector_config": ["credentials", 0, "client_secret"]
+        }
+      }
+    }
+  }
 }

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -10,6 +10,7 @@
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.oauth.flows.AsanaOAuthFlow;
 import io.airbyte.oauth.flows.GithubOAuthFlow;
+import io.airbyte.oauth.flows.HarvestOAuthFlow;
 import io.airbyte.oauth.flows.HubspotOAuthFlow;
 import io.airbyte.oauth.flows.IntercomOAuthFlow;
 import io.airbyte.oauth.flows.LinkedinAdsOAuthFlow;
@@ -45,6 +46,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final
         .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-google-sheets", new GoogleSheetsOAuthFlow(configRepository, httpClient))
+        .put("airbyte/source-harvest", new HarvestOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-hubspot", new HubspotOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-intercom", new IntercomOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient))

airbyte-oauth/src/main/java/io/airbyte/oauth/flows/HarvestOAuthFlow.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.oauth.flows;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.oauth.BaseOAuth2Flow;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import org.apache.http.client.utils.URIBuilder;
+
+public class HarvestOAuthFlow extends BaseOAuth2Flow {
+
+  private static final String AUTHORIZE_URL = "https://id.getharvest.com/oauth2/authorize";
+  private static final String ACCESS_TOKEN_URL = "https://id.getharvest.com/api/v2/oauth2/token";
+
+  public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
+    super(configRepository, httpClient);
+  }
+
+  public HarvestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
+    super(configRepository, httpClient, stateSupplier);
+  }
+
+  /**
+   * Depending on the OAuth flow implementation, the URL to grant user's consent may differ,
+   * especially in the query parameters to be provided. This function should generate such consent URL
+   * accordingly.
+   *
+   * @param definitionId The configured definition ID of this client
+   * @param clientId The configured client ID
+   * @param redirectUrl the redirect URL
+   */
+  @Override
+  protected String formatConsentUrl(final UUID definitionId,
+                                    final String clientId,
+                                    final String redirectUrl,
+                                    final JsonNode inputOAuthConfiguration)
+      throws IOException {
+    try {
+      return new URIBuilder(AUTHORIZE_URL)
+          .addParameter("client_id", clientId)
+          .addParameter("response_type", "code")
+          .addParameter("redirect_uri", redirectUrl)
+          .addParameter("state", getState())
+          .build().toString();
+    } catch (final URISyntaxException e) {
+      throw new IOException("Failed to format Consent URL for OAuth flow", e);
+    }
+  }
+
+  @Override
+  protected Map<String, String> getAccessTokenQueryParameters(final String clientId,
+                                                              final String clientSecret,
+                                                              final String authCode,
+                                                              final String redirectUrl) {
+    return ImmutableMap.<String, String>builder()
+        // required
+        .put("client_id", clientId)
+        .put("redirect_uri", redirectUrl)
+        .put("client_secret", clientSecret)
+        .put("code", authCode)
+        .put("grant_type", "authorization_code")
+        .build();
+  }
+
+  /**
+   * Returns the URL where to retrieve the access token from.
+   *
+   */
+  @Override
+  protected String getAccessTokenUrl() {
+    return ACCESS_TOKEN_URL;
+  }
+
+}

airbyte-oauth/src/test/java/io/airbyte/oauth/flows/HarvestOAuthFlowTest.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.oauth.flows;
+
+import io.airbyte.oauth.BaseOAuthFlow;
+
+public class HarvestOAuthFlowTest extends BaseOAuthFlowTest {
+
+  @Override
+  protected BaseOAuthFlow getOAuthFlow() {
+    return new HarvestOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState);
+  }
+
+  @Override
+  protected String getExpectedConsentUrl() {
+    return "https://id.getharvest.com/oauth2/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&state=state";
+  }
+
+}

docs/integrations/sources/harvest.md

@@ -69,6 +69,7 @@ See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support |
 | 0.1.5 | 2021-09-28 | [5747](https://github.com/airbytehq/airbyte/pull/5747) | Update schema date-time fields |
 | 0.1.4 | 2021-06-22 | [5701](https://github.com/airbytehq/airbyte/pull/5071) | Harvest normalization failure: fixing the schemas |
 | 0.1.3 | 2021-06-22 | [4274](https://github.com/airbytehq/airbyte/pull/4274) | Fix wrong data type on `statement_key` in `clients` stream |

tools/bin/ci_credentials.sh

@@ -224,7 +224,6 @@ read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.j
 read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS"
 read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS"
 read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json"
-read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS"
 read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS"
 read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json"
 read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS"