From b230bc5200f41550d692dc118ad3c03193227018 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 21 Oct 2021 16:40:03 +0300 Subject: [PATCH] Github oauth backend. (#7237) --- .../java/io/airbyte/oauth/BaseOAuthFlow.java | 5 +- .../oauth/OAuthImplementationFactory.java | 2 + .../airbyte/oauth/flows/AsanaOAuthFlow.java | 3 +- .../flows/FacebookMarketingOAuthFlow.java | 2 +- .../airbyte/oauth/flows/GithubOAuthFlow.java | 82 ++++++++++++++++++ .../oauth/flows/SalesforceOAuthFlow.java | 2 +- .../oauth/flows/google/GoogleOAuthFlow.java | 3 +- .../GithubOAuthFlowIntegrationTest.java | 84 +++++++++++++++++++ .../oauth/flows/OAuthFlowIntegrationTest.java | 13 ++- .../oauth/flows/GithubOAuthFlowTest.java | 82 ++++++++++++++++++ 10 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GithubOAuthFlow.java create mode 100644 airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/GithubOAuthFlowIntegrationTest.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java index ddf19647d961..def90e9fab5a 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java @@ -175,12 +175,13 @@ private Map completeOAuthFlow(final String clientId, final String redirectUrl, JsonNode oAuthParamConfig) throws IOException { - var accessTokenUrl = getAccessTokenUrl(oAuthParamConfig); + var accessTokenUrl = getAccessTokenUrl(); final HttpRequest request = HttpRequest.newBuilder() .POST(HttpRequest.BodyPublishers .ofString(tokenReqContentType.converter.apply(getAccessTokenQueryParameters(clientId, clientSecret, authCode, redirectUrl)))) .uri(URI.create(accessTokenUrl)) .header("Content-Type", tokenReqContentType.contentType) + .header("Accept", "application/json") .build(); // TODO: Handle error response to report better messages try { @@ -220,7 +221,7 @@ protected String extractCodeParameter(Map queryParams) throws IO /** * Returns the URL where to retrieve the access token from. */ - protected abstract String getAccessTokenUrl(JsonNode oAuthParamConfig); + protected abstract String getAccessTokenUrl(); protected Map extractRefreshToken(final JsonNode data, String accessTokenUrl) throws IOException { final Map result = new HashMap<>(); diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 218e2c59f2b2..5cb96e3b255d 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -8,6 +8,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.flows.AsanaOAuthFlow; import io.airbyte.oauth.flows.FacebookMarketingOAuthFlow; +import io.airbyte.oauth.flows.GithubOAuthFlow; import io.airbyte.oauth.flows.SalesforceOAuthFlow; import io.airbyte.oauth.flows.TrelloOAuthFlow; import io.airbyte.oauth.flows.google.GoogleAdsOAuthFlow; @@ -24,6 +25,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository) { OAUTH_FLOW_MAPPING = ImmutableMap.builder() .put("airbyte/source-asana", new AsanaOAuthFlow(configRepository)) .put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository)) + .put("airbyte/source-github", new GithubOAuthFlow(configRepository)) .put("airbyte/source-google-ads", new GoogleAdsOAuthFlow(configRepository)) .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOAuthFlow(configRepository)) .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/AsanaOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/AsanaOAuthFlow.java index 3a967c694cc8..9e273b82b133 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/AsanaOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/AsanaOAuthFlow.java @@ -4,7 +4,6 @@ package io.airbyte.oauth.flows; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; @@ -49,7 +48,7 @@ protected String formatConsentUrl(UUID definitionId, String clientId, String red } @Override - protected String getAccessTokenUrl(JsonNode oAuthParamConfig) { + protected String getAccessTokenUrl() { return ACCESS_TOKEN_URL; } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java index a710f01d1b12..7d3a2ca18528 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java @@ -54,7 +54,7 @@ protected String formatConsentUrl(final UUID definitionId, final String clientId } @Override - protected String getAccessTokenUrl(JsonNode oAuthParamConfig) { + protected String getAccessTokenUrl() { return ACCESS_TOKEN_URL; } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GithubOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GithubOAuthFlow.java new file mode 100644 index 000000000000..bb2e14ac11f6 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GithubOAuthFlow.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuthFlow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from + * https://docs.github.com/en/developers/apps/building-oauth-apps/authorizing-oauth-apps#web-application-flow + */ +public class GithubOAuthFlow extends BaseOAuthFlow { + + private static final String AUTHORIZE_URL = "https://github.com/login/oauth/authorize"; + private static final String ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token"; + + public GithubOAuthFlow(final ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + GithubOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl) throws IOException { + try { + // No scope means read-only access to public information + // https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes + return new URIBuilder(AUTHORIZE_URL) + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("state", getState()) + .build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map extractRefreshToken(final JsonNode data, String accessTokenUrl) throws IOException { + System.out.println(data); + if (data.has("access_token")) { + return Map.of("credentials", Map.of("access_token", data.get("access_token").asText())); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", ACCESS_TOKEN_URL)); + } + } + + @Override + protected String getClientIdUnsafe(final JsonNode config) { + // the config object containing client ID and secret is nested inside the "credentials" object + Preconditions.checkArgument(config.hasNonNull("credentials")); + return super.getClientIdUnsafe(config.get("credentials")); + } + + @Override + protected String getClientSecretUnsafe(final JsonNode config) { + // the config object containing client ID and secret is nested inside the "credentials" object + Preconditions.checkArgument(config.hasNonNull("credentials")); + return super.getClientSecretUnsafe(config.get("credentials")); + } + +} diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java index 5790999ba005..22fdb6dbd791 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/SalesforceOAuthFlow.java @@ -51,7 +51,7 @@ protected String formatConsentUrl(UUID definitionId, String clientId, String red } @Override - protected String getAccessTokenUrl(JsonNode oAuthConfig) { + protected String getAccessTokenUrl() { return ACCESS_TOKEN_URL; } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java index 497a633b8a7d..1c460101f854 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java @@ -4,7 +4,6 @@ package io.airbyte.oauth.flows.google; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; @@ -63,7 +62,7 @@ protected String formatConsentUrl(final UUID definitionId, final String clientId protected abstract String getScope(); @Override - protected String getAccessTokenUrl(JsonNode oAuthParamConfig) { + protected String getAccessTokenUrl() { return ACCESS_TOKEN_URL; } diff --git a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/GithubOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/GithubOAuthFlowIntegrationTest.java new file mode 100644 index 000000000000..7d569291c304 --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/GithubOAuthFlowIntegrationTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.OAuthFlowImplementation; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class GithubOAuthFlowIntegrationTest extends OAuthFlowIntegrationTest { + + protected static final Path CREDENTIALS_PATH = Path.of("secrets/github.json"); + protected static final String REDIRECT_URL = "http://localhost:8000/auth_flow"; + protected static final int SERVER_LISTENING_PORT = 8000; + + @Override + protected Path get_credentials_path() { + return CREDENTIALS_PATH; + } + + @Override + protected OAuthFlowImplementation getFlowObject(ConfigRepository configRepository) { + return new GithubOAuthFlow(configRepository); + } + + @Override + protected int getServerListeningPort() { + return SERVER_LISTENING_PORT; + } + + @BeforeEach + public void setup() throws IOException { + super.setup(); + } + + @Test + public void testFullGithubOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .put("client_id", credentialsJson.get("client_id").asText()) + .put("client_secret", credentialsJson.get("client_secret").asText()) + .build())))); + final String url = flow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = flow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("access_token")); + assertTrue(params.get("access_token").toString().length() > 0); + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/OAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/OAuthFlowIntegrationTest.java index c9ac5fb199fe..138604ab074f 100644 --- a/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/OAuthFlowIntegrationTest.java +++ b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/OAuthFlowIntegrationTest.java @@ -30,7 +30,8 @@ public abstract class OAuthFlowIntegrationTest { * due to the consent flow in the browser */ protected static final Logger LOGGER = LoggerFactory.getLogger(OAuthFlowIntegrationTest.class); - protected static final String REDIRECT_URL = "http://localhost/code"; + protected static final String REDIRECT_URL = "http://localhost/auth_flow"; + protected static final int SERVER_LISTENING_PORT = 80; protected ConfigRepository configRepository; protected OAuthFlowImplementation flow; @@ -51,14 +52,20 @@ public void setup() throws IOException { flow = this.getFlowObject(configRepository); - server = HttpServer.create(new InetSocketAddress(80), 0); + System.out.println(getServerListeningPort()); + server = HttpServer.create(new InetSocketAddress(getServerListeningPort()), 0); server.setExecutor(null); // creates a default executor server.start(); serverHandler = new ServerHandler("code"); - server.createContext("/code", serverHandler); + // Same endpoint as we use for airbyte instance + server.createContext("/auth_flow", serverHandler); } + protected int getServerListeningPort() { + return SERVER_LISTENING_PORT; + } + @AfterEach void tearDown() { server.stop(1); diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java new file mode 100644 index 000000000000..84e6858f7c33 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/GithubOAuthFlowTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class GithubOAuthFlowTest { + + private UUID workspaceId; + private UUID definitionId; + private ConfigRepository configRepository; + private GithubOAuthFlow githuboAuthFlow; + private HttpClient httpClient; + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private static String getConstantState() { + return "state"; + } + + @BeforeEach + public void setup() throws IOException, JsonValidationException { + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + configRepository = mock(ConfigRepository.class); + httpClient = mock(HttpClient.class); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode( + Map.of("credentials", + ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()))))); + githuboAuthFlow = new GithubOAuthFlow(configRepository, httpClient, GithubOAuthFlowTest::getConstantState); + + } + + @Test + public void testGetSourceConcentUrl() throws IOException, InterruptedException, ConfigNotFoundException { + final String concentUrl = + githuboAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + assertEquals(concentUrl, + "https://github.com/login/oauth/authorize?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&state=state"); + } + + @Test + public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { + + Map returnedCredentials = Map.of("access_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + githuboAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); + } + +}