diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 9d29fad98694..0d0e3836dcd6 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -106,6 +106,8 @@ jobs: GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }} + GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }} + GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 602ac9a53b21..d492afc8464b 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -101,6 +101,8 @@ jobs: GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }} + GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }} + GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json index aedaa4e4c700..440e87533b38 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", "name": "Google Analytics v4", "dockerRepository": "airbyte/source-google-analytics-v4", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.7", "documentationUrl": "https://docs.airbyte.io/integrations/sources/source-google-analytics-v4", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 61324d7fade7..d6ea388add06 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -140,7 +140,7 @@ - sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 name: Google Analytics v4 dockerRepository: airbyte/source-google-analytics-v4 - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/source-google-analytics-v4 icon: google-analytics.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index 4c0d310f62a5..1f93e5aab0cb 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml index bd39651e237d..cec49c42616d 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml @@ -7,18 +7,22 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/service_config.json" + status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: - - config_path: "secrets/config.json" + - config_path: "secrets/service_config.json" basic_read: - - config_path: "secrets/config.json" + - config_path: "secrets/service_config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] incremental: - - config_path: "secrets/config.json" + - config_path: "secrets/service_config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" + - config_path: "secrets/service_config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json index 93c0ce5c25a4..0ab7ad4763b3 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json @@ -1,5 +1,8 @@ { - "credentials": { "credentials_json": "" }, + "credentials": { + "auth_type": "Service", + "credentials_json": "None" + }, "view_id": "211669975", "start_date": "2021-02-11", "window_in_days": 1, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index 02925e3ca559..254dac20aedc 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -383,55 +383,29 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} -class GoogleAnalyticsOauth2Authenticator(Oauth2Authenticator): - """ - This class supports either default authorization_code and JWT OAuth - authorizations in case of service account. - - Request example for API token extraction: +class GoogleAnalyticsServiceOauth2Authenticator(Oauth2Authenticator): + """Request example for API token extraction: curl --location --request POST https://oauth2.googleapis.com/token?grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=signed_JWT """ - use_jwt_auth: bool = False - def __init__(self, config): - client_secret, client_id, refresh_token = None, None, None - if "credentials_json" in config: - # Backward compatability with previous config format. Use - # credentials_json from config root. - auth = config - else: - auth = config["credentials"] - if "credentials_json" in auth: - # Service account JWT authorization - self.use_jwt_auth = True - credentials_json = json.loads(auth["credentials_json"]) - client_secret, client_id, refresh_token = credentials_json["private_key"], credentials_json["private_key_id"], None - self.client_email = credentials_json["client_email"] - else: - # OAuth 2.0 authorization_code authorization - client_secret, client_id, refresh_token = auth["client_secret"], auth["client_id"], auth["refresh_token"] + self.credentials_json = json.loads(config["credentials_json"]) + self.client_email = self.credentials_json["client_email"] self.scope = "https://www.googleapis.com/auth/analytics.readonly" super().__init__( token_refresh_endpoint="https://oauth2.googleapis.com/token", - client_secret=client_secret, - client_id=client_id, - refresh_token=refresh_token, - scopes=[self.scope], + client_secret=self.credentials_json["private_key"], + client_id=self.credentials_json["private_key_id"], + refresh_token=None, ) def refresh_access_token(self) -> Tuple[str, int]: """ - Calling the Google OAuth 2.0 token endpoint. Used for authorizing - with signed JWT if credentials_json provided by config. Otherwise use - default OAuth2.0 workflow. - :return tuple with access token and token's time-to-live. + Calling the Google OAuth 2.0 token endpoint. Used for authorizing signed JWT. + Returns tuple with access token and token's time-to-live """ - if not self.use_jwt_auth: - return super().refresh_access_token() - response_json = None try: response = requests.request(method="POST", url=self.token_refresh_endpoint, params=self.get_refresh_request_params()) @@ -452,7 +426,6 @@ def refresh_access_token(self) -> Tuple[str, int]: def get_refresh_request_params(self) -> Mapping[str, any]: """ Sign the JWT with RSA-256 using the private key found in service account JSON file. - Not used with default OAuth2.0 authorization_code grant_type. """ token_lifetime = 3600 # token lifetime is 1 hour @@ -469,17 +442,35 @@ def get_refresh_request_params(self) -> Mapping[str, any]: } headers = {"kid": self.client_id} signed_jwt = jwt.encode(payload, self.client_secret, headers=headers, algorithm="RS256") - return {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": signed_jwt} + return {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": str(signed_jwt)} class SourceGoogleAnalyticsV4(AbstractSource): """Google Analytics lets you analyze data about customer engagement with your website or application.""" + @staticmethod + def get_authenticator(config): + # backwards compatibility, credentials_json used to be in the top level of the connector + if config.get("credentials_json"): + return GoogleAnalyticsServiceOauth2Authenticator(config) + + auth_params = config.get("credentials") + if auth_params.pop("auth_type") == "Service": + return GoogleAnalyticsServiceOauth2Authenticator(auth_params) + else: + return Oauth2Authenticator( + token_refresh_endpoint="https://oauth2.googleapis.com/token", + client_secret=auth_params.get("client_secret"), + client_id=auth_params.get("client_id"), + refresh_token=auth_params.get("refresh_token"), + scopes=["https://www.googleapis.com/auth/analytics.readonly"], + ) + def check_connection(self, logger, config) -> Tuple[bool, any]: try: url = f"{GoogleAnalyticsV4TypesList.url_base}" - authenticator = GoogleAnalyticsOauth2Authenticator(config) + authenticator = self.get_authenticator(config) session = requests.get(url, headers=authenticator.get_auth_header()) session.raise_for_status() @@ -496,7 +487,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: streams: List[GoogleAnalyticsV4Stream] = [] - authenticator = GoogleAnalyticsOauth2Authenticator(config) + authenticator = self.get_authenticator(config) config["authenticator"] = authenticator diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index 2d9dfa8d9f61..c7de4cd7a339 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -4,50 +4,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Analytics V4 Spec", "type": "object", - "required": ["credentials", "view_id", "start_date"], + "required": ["view_id", "start_date"], "additionalProperties": true, "properties": { - "credentials": { - "title": "Authentication mechanism", - "type": "object", - "description": "Choose either OAuth2.0 flow or provide your own JWT credentials for service account", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0 authorization", - "properties": { - "option_title": { - "type": "string", - "const": "Default OAuth2.0 authorization" - }, - "client_id": { "type": "string" }, - "client_secret": { "type": "string", "airbyte_secret": true }, - "refresh_token": { "type": "string", "airbyte_secret": true }, - "access_token": { "type": "string", "airbyte_secret": true } - }, - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": false - }, - { - "type": "object", - "title": "Service Account Key", - "properties": { - "option_title": { - "type": "string", - "const": "Service account credentials" - }, - "credentials_json": { - "type": "string", - "title": "Credentials JSON", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", - "airbyte_secret": true - } - }, - "required": ["credentials_json"], - "additionalProperties": true - } - ] - }, "view_id": { "type": "string", "title": "View ID", @@ -70,6 +29,75 @@ "title": "Custom Reports", "type": "string", "description": "A JSON array describing the custom reports you want to sync from GA. Check out the docs to get more information about this field." + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "title": "Authenticate via Google (Oauth)", + "type": "object", + "required": [ + "auth_type", + "client_id", + "client_secret", + "refresh_token" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "Client", + "enum": ["Client"], + "default": "Client", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your developer application", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The client secret of your developer application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "A refresh token generated using the above client ID and secret", + "airbyte_secret": true + }, + "access_token": { + "title": "Access Token", + "type": "string", + "description": "A access token generated using the above client ID, secret and refresh_token", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "Service Account Key Authentication", + "required": ["auth_type", "credentials_json"], + "properties": { + "auth_type": { + "type": "string", + "const": "Service", + "enum": ["Service"], + "default": "Service", + "order": 0 + }, + "credentials_json": { + "type": "string", + "description": "The JSON key of the service account to use for authorization", + "examples": [ + "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" + ] + } + } + } + ] } } }, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index 84e0ee4b077a..62a574688692 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -69,7 +69,10 @@ def test_lookup_metrics_dimensions_data_type(metrics_dimensions_mapping, mock_me def test_check_connection_jwt(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call): test_config = json.loads(read_file("../integration_tests/sample_config.json")) del test_config["custom_reports"] - test_config["credentials"] = {"credentials_json": '{"client_email": "", "private_key": "", "private_key_id": ""}'} + test_config["credentials"] = { + "auth_type": "Service", + "credentials_json": '{"client_email": "", "private_key": "", "private_key_id": ""}', + } source = SourceGoogleAnalyticsV4() assert source.check_connection(MagicMock(), test_config) == (True, None) jwt_encode_mock.encode.assert_called() @@ -81,6 +84,7 @@ def test_check_connection_oauth(jwt_encode_mock, mocker, mock_metrics_dimensions test_config = json.loads(read_file("../integration_tests/sample_config.json")) del test_config["custom_reports"] test_config["credentials"] = { + "auth_type": "Client", "client_id": "client_id_val", "client_secret": "client_secret_val", "refresh_token": "refresh_token_val", diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index c83db00da8c0..5a5cf6c57d7b 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -131,7 +131,8 @@ The Google Analytics connector should not run into Google Analytics API limitati | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.6 | 2021-09-27 | [6459](https://github.com/airbytehq/airbyte/pull/6459) | Update OAuth Spec File | +| 0.1.7 | 2021-10-07 | [6414](https://github.com/airbytehq/airbyte/pull/6414) | Declare oauth parameters in google sources | +| 0.1.6 | 2021-09-27 | [6459](https://github.com/airbytehq/airbyte/pull/6459) | Update OAuth Spec File | | 0.1.3 | 2021-09-21 | [6357](https://github.com/airbytehq/airbyte/pull/6357) | Fix oauth workflow parameters | | 0.1.2 | 2021-09-20 | [6306](https://github.com/airbytehq/airbyte/pull/6306) | Support of airbyte OAuth initialization flow | | 0.1.1 | 2021-08-25 | [5655](https://github.com/airbytehq/airbyte/pull/5655) | Corrected validation of empty custom report| diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 033695e4138b..34b7ed1cfb4b 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -75,6 +75,8 @@ write_standard_creds source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS" write_standard_creds source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS" write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS" write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS" +write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" +write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json"