From 84534b7eddb6b761c541f5738149849094454a6b Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 22 Apr 2022 14:58:52 +0530 Subject: [PATCH 1/5] Added support of eu endpoint. --- tap_pendo/streams.py | 14 ++++++++++---- tests/unittests/test_endpoints_dict.py | 11 +++++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index 8310058..7ef18f9 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -20,7 +20,8 @@ from tap_pendo import utils as tap_pendo_utils KEY_PROPERTIES = ['id'] -BASE_URL = "https://app.pendo.io" +US_BASE_URL = "https://app.pendo.io" +EU_BASE_URL = "https://app.eu.pendo.io" LOGGER = singer.get_logger() session = requests.Session() @@ -92,11 +93,16 @@ def __init__(self, endpoint, method, headers=None, params=None): self.headers = headers self.params = params - def get_url(self, **kwargs): + def get_url(self, is_eu_domain, **kwargs): """ Concatenate and format the dynamic values to the BASE_URL """ - return BASE_URL + self.endpoint.format(**kwargs) + # Update url if `eu` domain is selected + if str(is_eu_domain).lower() == "true": + return EU_BASE_URL + self.endpoint.format(**kwargs) + else: + return US_BASE_URL + self.endpoint.format(**kwargs) + class Stream(): @@ -156,7 +162,7 @@ def request(self, endpoint, params=None, **kwargs): } request_kwargs = { - 'url': self.endpoint.get_url(**kwargs), + 'url': self.endpoint.get_url(self.config.get('eu_domain', False), **kwargs), 'method': self.endpoint.method, 'headers': headers, 'params': params diff --git a/tests/unittests/test_endpoints_dict.py b/tests/unittests/test_endpoints_dict.py index 0068da6..2660178 100644 --- a/tests/unittests/test_endpoints_dict.py +++ b/tests/unittests/test_endpoints_dict.py @@ -14,5 +14,12 @@ def test_correct_values_passed_in_endpoint_object(): def test_correct_endpoint_url(): stream.endpoint = Endpoints( "/api/v1/visitor/{visitorID}/history", "GET") - formatted_url = stream.endpoint.get_url(visitorID='abc') - assert formatted_url == 'https://app.pendo.io/api/v1/visitor/abc/history' \ No newline at end of file + formatted_url = stream.endpoint.get_url(visitorID='abc', is_eu_domain=False) # Assign is_eu_domain=False for US region endpoint + assert formatted_url == 'https://app.pendo.io/api/v1/visitor/abc/history' + +def test_eu_endpoint_url(): + stream.endpoint = Endpoints( + "/api/v1/visitor/{visitorID}/history", "GET") + formatted_url = stream.endpoint.get_url(visitorID='abc', is_eu_domain=True) + # Verify formatted_url for eu endpoint + assert formatted_url == 'https://app.eu.pendo.io/api/v1/visitor/abc/history' \ No newline at end of file From 4221978b1a25880435b491186b8e90791eaa0dfb Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 22 Apr 2022 15:42:19 +0530 Subject: [PATCH 2/5] Updated readme file --- README.md | 54 ++++++++++++++++++++++++++++++++-------------- sample_config.json | 3 ++- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b8cc15c..35c1cda 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: - Pulls raw data from the [Pendo API](https://developers.pendo.io/docs/?bash#overview). +- Supports following two server + - US Server + - EU Server - Extracts the following resources: - Accounts - Features @@ -37,7 +40,8 @@ This tap: **[accounts](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `account_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `lastupdated` @@ -48,7 +52,8 @@ This tap: **[features](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/feature) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_updated_at` @@ -56,7 +61,8 @@ This tap: **[guides](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -64,7 +70,8 @@ This tap: **[track types](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -72,7 +79,8 @@ This tap: **[visitors](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `lastupdated` @@ -86,7 +94,8 @@ This tap: **[visitor_history](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `modified_ts` (Max from `ts` or `lastTs`) @@ -94,7 +103,8 @@ This tap: **[feature_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -102,7 +112,8 @@ This tap: **[events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -110,7 +121,8 @@ This tap: **[page_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -118,7 +130,8 @@ This tap: **[guide_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` @@ -126,7 +139,8 @@ This tap: **[poll_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` @@ -134,7 +148,8 @@ This tap: **[track_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -142,7 +157,8 @@ This tap: **[guides](https://developers.pendo.io/docs/?bash#entities)** -- Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -150,13 +166,15 @@ This tap: **[metadata accounts](https://developers.pendo.io/docs/?bash#automatically-generated-metadata)** -- Endpoint: [https://api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) +- US Server Endpoint: [https://app.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) - Replication strategy: FULL_TABLE - Transformations: Camel to snake case. **[metadata visitors](https://developers.pendo.io/docs/?bash#automatically-generated-metadata)** -- Endpoint: [https://api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/visitor) +- US Server Endpoint: [https://app.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) +- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) - Replication strategy: FULL_TABLE - Transformations: Camel to snake case. @@ -164,6 +182,8 @@ This tap: Authentication is managed by integration keys. An integration key may be created in the Pendo website: Settings -> Integrations -> Integration Keys. +### Note +- If you selected eu_domain then please make sure you enter integration_key of that project only. ## State ```json @@ -246,6 +266,7 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during - `period` (string, `ABCdef123`): `dayRange` or `hourRange` - `lookback_window` (integer): 10 (For event objects. Default: 0) - `request_timeout` (integer): 300 (For passing timeout to the request. Default: 300) + - `eu_domain` (boolean, `true` or `false`): EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records. ```json { @@ -254,7 +275,8 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during "period": "dayRange", "lookback_window": 10, "request_timeout": 300, - "include_anonymous_visitors": "true" + "include_anonymous_visitors": "true", + "eu_domain": "true" } ``` diff --git a/sample_config.json b/sample_config.json index f35a1b2..b4776eb 100644 --- a/sample_config.json +++ b/sample_config.json @@ -1,4 +1,5 @@ { "x_pendo_integration_key": "x_pendo_integration_key", - "start_date": "2017-01-01T00:00:00Z" + "start_date": "2017-01-01T00:00:00Z", + "eu_domain": "true" } \ No newline at end of file From 0652028aa6ade6ea5fc8f81b228bd7fcf107bcce Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 22 Apr 2022 16:03:45 +0530 Subject: [PATCH 3/5] Updated readme file --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35c1cda..f857672 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ This tap: Authentication is managed by integration keys. An integration key may be created in the Pendo website: Settings -> Integrations -> Integration Keys. ### Note -- If you selected eu_domain then please make sure you enter integration_key of that project only. +- If you selected eu_domain then please make sure you enter integration_key of Pendo EU subscription only. ## State ```json @@ -266,7 +266,7 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during - `period` (string, `ABCdef123`): `dayRange` or `hourRange` - `lookback_window` (integer): 10 (For event objects. Default: 0) - `request_timeout` (integer): 300 (For passing timeout to the request. Default: 300) - - `eu_domain` (boolean, `true` or `false`): EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records. + - `eu_domain` (boolean, `true` or `false`): EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records else it will use the US endpoint to capture the records. It is an optional property. ```json { From 3a9b66f29b43d67d4c89b9fbf5dc253c396a6425 Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 22 Apr 2022 16:21:56 +0530 Subject: [PATCH 4/5] Updated readme for eu_domain property --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f857672..f793f2e 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during - `period` (string, `ABCdef123`): `dayRange` or `hourRange` - `lookback_window` (integer): 10 (For event objects. Default: 0) - `request_timeout` (integer): 300 (For passing timeout to the request. Default: 300) - - `eu_domain` (boolean, `true` or `false`): EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records else it will use the US endpoint to capture the records. It is an optional property. + - `eu_domain` (boolean, `true` or `false`): It is an optional property. EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records else it will use the US endpoint by default to capture the records. ```json { From 757473793a7d5e980b216fbc1c5648abe31cb9c8 Mon Sep 17 00:00:00 2001 From: prijendev Date: Mon, 25 Apr 2022 11:21:22 +0530 Subject: [PATCH 5/5] Updated eu endpoint support logic --- README.md | 74 ++++++++++++-------------- sample_config.json | 3 +- tap_pendo/streams.py | 8 +-- tests/unittests/test_endpoints_dict.py | 6 +-- 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index f793f2e..94043fd 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: - Pulls raw data from the [Pendo API](https://developers.pendo.io/docs/?bash#overview). -- Supports following two server - - US Server - - EU Server +- Supports following two subscription + - US Subscription + - EU Subscription - Extracts the following resources: - Accounts - Features @@ -40,8 +40,8 @@ This tap: **[accounts](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `account_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `lastupdated` @@ -52,8 +52,8 @@ This tap: **[features](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_updated_at` @@ -61,8 +61,8 @@ This tap: **[guides](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -70,8 +70,8 @@ This tap: **[track types](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -79,8 +79,8 @@ This tap: **[visitors](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `lastupdated` @@ -94,8 +94,8 @@ This tap: **[visitor_history](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `modified_ts` (Max from `ts` or `lastTs`) @@ -103,8 +103,8 @@ This tap: **[feature_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -112,8 +112,8 @@ This tap: **[events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -121,8 +121,8 @@ This tap: **[page_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -130,8 +130,8 @@ This tap: **[guide_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` @@ -139,8 +139,8 @@ This tap: **[poll_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` @@ -148,8 +148,8 @@ This tap: **[track_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` @@ -157,8 +157,8 @@ This tap: **[guides](https://developers.pendo.io/docs/?bash#entities)** -- US Server Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/aggregation](https://app.eu.pendo.io/api/v1/aggregation) - Primary key fields: `id` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `last_pdated_at` @@ -166,15 +166,15 @@ This tap: **[metadata accounts](https://developers.pendo.io/docs/?bash#automatically-generated-metadata)** -- US Server Endpoint: [https://app.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/account](https://app.pendo.io/api/v1/metadata/schema/account) - Replication strategy: FULL_TABLE - Transformations: Camel to snake case. **[metadata visitors](https://developers.pendo.io/docs/?bash#automatically-generated-metadata)** -- US Server Endpoint: [https://app.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) -- EU Server Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) +- US Subscription Endpoint: [https://app.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) +- EU Subscription Endpoint: [https://app.eu.pendo.io/api/v1/metadata/schema/visitor](https://app.pendo.io/api/v1/metadata/schema/visitor) - Replication strategy: FULL_TABLE - Transformations: Camel to snake case. @@ -182,8 +182,6 @@ This tap: Authentication is managed by integration keys. An integration key may be created in the Pendo website: Settings -> Integrations -> Integration Keys. -### Note -- If you selected eu_domain then please make sure you enter integration_key of Pendo EU subscription only. ## State ```json @@ -266,7 +264,6 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during - `period` (string, `ABCdef123`): `dayRange` or `hourRange` - `lookback_window` (integer): 10 (For event objects. Default: 0) - `request_timeout` (integer): 300 (For passing timeout to the request. Default: 300) - - `eu_domain` (boolean, `true` or `false`): It is an optional property. EU domain refers to the physical storage location of an organization's data or information. Setting this config parameter to true ensures that it uses the EU endpoint to capture the records else it will use the US endpoint by default to capture the records. ```json { @@ -275,8 +272,7 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during "period": "dayRange", "lookback_window": 10, "request_timeout": 300, - "include_anonymous_visitors": "true", - "eu_domain": "true" + "include_anonymous_visitors": "true" } ``` @@ -380,4 +376,4 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during --- -Copyright © 2020 Stitch +Copyright © 2020 Stitch \ No newline at end of file diff --git a/sample_config.json b/sample_config.json index b4776eb..f35a1b2 100644 --- a/sample_config.json +++ b/sample_config.json @@ -1,5 +1,4 @@ { "x_pendo_integration_key": "x_pendo_integration_key", - "start_date": "2017-01-01T00:00:00Z", - "eu_domain": "true" + "start_date": "2017-01-01T00:00:00Z" } \ No newline at end of file diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index 7ef18f9..e3d579e 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -93,12 +93,12 @@ def __init__(self, endpoint, method, headers=None, params=None): self.headers = headers self.params = params - def get_url(self, is_eu_domain, **kwargs): + def get_url(self, integration_key, **kwargs): """ Concatenate and format the dynamic values to the BASE_URL """ - # Update url if `eu` domain is selected - if str(is_eu_domain).lower() == "true": + # Update url if integration_key ends with `.eu`. + if integration_key[-3:] == ".eu": return EU_BASE_URL + self.endpoint.format(**kwargs) else: return US_BASE_URL + self.endpoint.format(**kwargs) @@ -162,7 +162,7 @@ def request(self, endpoint, params=None, **kwargs): } request_kwargs = { - 'url': self.endpoint.get_url(self.config.get('eu_domain', False), **kwargs), + 'url': self.endpoint.get_url(self.config['x_pendo_integration_key'], **kwargs), 'method': self.endpoint.method, 'headers': headers, 'params': params diff --git a/tests/unittests/test_endpoints_dict.py b/tests/unittests/test_endpoints_dict.py index 2660178..3e03681 100644 --- a/tests/unittests/test_endpoints_dict.py +++ b/tests/unittests/test_endpoints_dict.py @@ -14,12 +14,12 @@ def test_correct_values_passed_in_endpoint_object(): def test_correct_endpoint_url(): stream.endpoint = Endpoints( "/api/v1/visitor/{visitorID}/history", "GET") - formatted_url = stream.endpoint.get_url(visitorID='abc', is_eu_domain=False) # Assign is_eu_domain=False for US region endpoint + formatted_url = stream.endpoint.get_url(visitorID='abc', integration_key="rf4007a20459.us") # Pass integration_key of US subscription account assert formatted_url == 'https://app.pendo.io/api/v1/visitor/abc/history' def test_eu_endpoint_url(): stream.endpoint = Endpoints( "/api/v1/visitor/{visitorID}/history", "GET") - formatted_url = stream.endpoint.get_url(visitorID='abc', is_eu_domain=True) - # Verify formatted_url for eu endpoint + formatted_url = stream.endpoint.get_url(visitorID='abc', integration_key="rf4007a20459.eu") + # Verify formatted_url for eu subscription account by integration_key. assert formatted_url == 'https://app.eu.pendo.io/api/v1/visitor/abc/history' \ No newline at end of file