From 93723d8fef9d88bfc4e6bdb567b0f0ba711b10aa Mon Sep 17 00:00:00 2001 From: harshpatel4_crest Date: Mon, 22 Nov 2021 15:10:59 +0530 Subject: [PATCH 1/6] updated PK for poll events --- tap_pendo/streams.py | 2 +- tests/tap_tester/base.py | 2 +- .../unittests/test_poll_events_primary_key.py | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/test_poll_events_primary_key.py diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index ef0bea6..2682a7e 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -636,7 +636,7 @@ def get_body(self, period, first): class PollEvents(Stream): replication_method = "INCREMENTAL" name = "poll_events" - key_properties = ['visitor_id', 'account_id', 'server_name', 'remote_ip'] + key_properties = ['visitor_id', 'account_id', 'poll_id', 'browser_time'] def __init__(self, config): super().__init__(config=config) diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py index 5d65360..4ff4776 100644 --- a/tests/tap_tester/base.py +++ b/tests/tap_tester/base.py @@ -99,7 +99,7 @@ def expected_metadata(self): self.REPLICATION_KEYS: {'browser_time'} }, "poll_events":{ - self.PRIMARY_KEYS: {"visitor_id", "account_id", "server_name", "remote_ip"}, + self.PRIMARY_KEYS: {"visitor_id", "account_id", "poll_id", "browser_time"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {'browser_time'} }, diff --git a/tests/unittests/test_poll_events_primary_key.py b/tests/unittests/test_poll_events_primary_key.py new file mode 100644 index 0000000..73e4369 --- /dev/null +++ b/tests/unittests/test_poll_events_primary_key.py @@ -0,0 +1,19 @@ +import unittest +from tap_pendo.streams import PollEvents + +class TestPollEventsPrimaryKey(unittest.TestCase): + + def test_poll_event_primary_key(self): + ''' + Verify that primary keys for 'poll_events' stream + ''' + # initialize config + config = {} + # expected primary key + expected_primary_keys = ['visitor_id', 'account_id', 'poll_id', 'browser_time'] + + # Initialize PollEvents object which sets primary keys + poll_events = PollEvents(config) + + # verify the Primary Key + self.assertEqual(poll_events.key_properties, expected_primary_keys) From aabdbd8212a27da27d01d2ba6e3bbf9b1c0d31cc Mon Sep 17 00:00:00 2001 From: harshpatel4_crest Date: Wed, 1 Dec 2021 15:28:33 +0530 Subject: [PATCH 2/6] updated readme file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b8cc15c..ca80f5f 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ This tap: **[poll_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` +- Primary key fields: `visitor_id`, `account_id`, `poll_id`, `browser_time` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` - Transformations: Camel to snake case. From 6a83f900db0cf6c8163d2172a87e2b360ff1bcad Mon Sep 17 00:00:00 2001 From: savan-chovatiya <80703490+savan-chovatiya@users.noreply.github.com> Date: Thu, 9 Dec 2021 11:22:07 +0530 Subject: [PATCH 3/6] Tdl 15317 fix primary key of feature events (#82) * TDL-15317: Updated primary key for feature-events * Moved integration test * TDL-15317: Updated integration test * updated readme file * Resolved internal PR review comments * Resolved review comments * run bookmark test with hour and day range Co-authored-by: harshpatel4_crest Co-authored-by: Harsh <80324346+harshpatel4crest@users.noreply.github.com> --- .circleci/config.yml | 2 +- README.md | 2 +- tap_pendo/streams.py | 6 +++- tests/tap_tester/base.py | 14 +++++---- tests/tap_tester/test_bookmark.py | 13 ++++++-- tests/tap_tester/test_discovery.py | 12 +++++++- .../test_feature_events_primary_keys.py | 30 +++++++++++++++++++ 7 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 tests/unittests/test_feature_events_primary_keys.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 79c785a..d3d0c72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ jobs: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-pendo/bin/activate - pip install coverage + pip install nose coverage nosetests --with-coverage --cover-erase --cover-package=tap_pendo --cover-html-dir=htmlcov tests/unittests coverage html - store_test_results: diff --git a/README.md b/README.md index b8cc15c..bf25887 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ This tap: **[feature_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` +- Primary key fields: `feature_id`, `visitor_id`, `account_id`, `server`, `remote_ip`, `user_agent`, `day` or `hour` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` - Transformations: Camel to snake case. diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index ef0bea6..11a26a2 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -551,7 +551,11 @@ def get_body(self): class FeatureEvents(EventsBase): name = "feature_events" replication_method = "INCREMENTAL" - key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip'] + key_properties = ['feature_id', 'visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + + def __init__(self, config): + super().__init__(config=config) + self.key_properties.append("day" if self.period == 'dayRange' else "hour") def get_body(self, key_id, period, first): return { diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py index 5d65360..67ee4f4 100644 --- a/tests/tap_tester/base.py +++ b/tests/tap_tester/base.py @@ -22,6 +22,7 @@ class TestPendoBase(unittest.TestCase): START_DATE_FORMAT = "%Y-%m-%dT00:00:00Z" BOOKMARK_COMPARISON_FORMAT = "%Y-%m-%dT%H:%M%S%z" start_date = "" + is_day_range = True @staticmethod def name(): @@ -39,6 +40,7 @@ def get_type(): def expected_metadata(self): """The expected streams and metadata about the streams""" + event_replication_key = 'day' if self.is_day_range else 'hour' return { "accounts": { self.PRIMARY_KEYS: {'account_id'}, @@ -79,19 +81,19 @@ def expected_metadata(self): self.REPLICATION_KEYS: {'last_updated_at'} }, "feature_events":{ - self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, + self.PRIMARY_KEYS: {"feature_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", event_replication_key}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {'day'} + self.REPLICATION_KEYS: {event_replication_key} }, "events": { self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {'day'} + self.REPLICATION_KEYS: {event_replication_key} }, "page_events": { self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {'day'} + self.REPLICATION_KEYS: {event_replication_key} }, "guide_events": { self.PRIMARY_KEYS: {"visitor_id", "account_id", "server_name", "remote_ip"}, @@ -106,7 +108,7 @@ def expected_metadata(self): "track_events": { self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {'day'} + self.REPLICATION_KEYS: {event_replication_key} }, "metadata_accounts": { self.REPLICATION_METHOD: self.FULL_TABLE, @@ -136,7 +138,7 @@ def get_properties(self, original: bool = True): return_value = { "start_date": "2020-09-10T00:00:00Z", "lookback_window": "1", - "period": "dayRange", + "period": "dayRange" if self.is_day_range else "hourRange", } if original: return return_value diff --git a/tests/tap_tester/test_bookmark.py b/tests/tap_tester/test_bookmark.py index d86a7e8..d27ac88 100644 --- a/tests/tap_tester/test_bookmark.py +++ b/tests/tap_tester/test_bookmark.py @@ -9,7 +9,7 @@ class PendoBookMarkTest(TestPendoBase): def name(self): return "pendo_bookmark_test" - def test_run(self): + def run_test(self): """ Verify that for each stream you can do a sync which records bookmarks. That the bookmark is the maximum value sent to the target for the replication key. @@ -191,4 +191,13 @@ def test_run(self): # Verify at least 1 record was replicated in the second sync self.assertGreater( - second_sync_count, 0, msg="We are not fully testing bookmarking for {}".format(stream)) \ No newline at end of file + second_sync_count, 0, msg="We are not fully testing bookmarking for {}".format(stream)) + + def test_run(self): + # test for hourRange period + self.is_day_range = False + self.run_test() + + # test for dayRange period + self.is_day_range = True + self.run_test() diff --git a/tests/tap_tester/test_discovery.py b/tests/tap_tester/test_discovery.py index dd43867..baf85f9 100644 --- a/tests/tap_tester/test_discovery.py +++ b/tests/tap_tester/test_discovery.py @@ -23,7 +23,7 @@ class PendoDiscoverTest(TestPendoBase): def name(self): return "pendo_discover_test" - def test_run(self): + def discovery_test_run(self): streams_to_test = self.expected_streams() conn_id = connections.ensure_connection(self, payload_hook=None) @@ -123,3 +123,13 @@ def test_run(self): and item.get("breadcrumb", ["properties", None])[1] not in actual_automatic_fields}), msg="Not all non key properties are set to available in metadata") + + def test_run(self): + + #Discovery test for hourRange period + self.is_day_range = False + self.discovery_test_run() + + #Discovery test for dayRange period + self.is_day_range = True + self.discovery_test_run() diff --git a/tests/unittests/test_feature_events_primary_keys.py b/tests/unittests/test_feature_events_primary_keys.py new file mode 100644 index 0000000..d99b008 --- /dev/null +++ b/tests/unittests/test_feature_events_primary_keys.py @@ -0,0 +1,30 @@ +import unittest +from tap_pendo.streams import FeatureEvents + +class TestFeatureEventsPrimaryKeys(unittest.TestCase): + + def test_feature_event_primary_key_with_hourRange(self): + ''' + Verify that primary keys should have expected fields with 'hour' field when period is hourRange + ''' + # Reset key properties to default value + FeatureEvents.key_properties = ['feature_id', 'visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + config = {"period": "hourRange"} # set hourRange as a period + expected_primary_keys = ["feature_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", "hour"] + + feature_event_stream1 = FeatureEvents(config) # Initialize FeatuereEvents object which sets primary keys + + self.assertEqual(feature_event_stream1.key_properties, expected_primary_keys) + + def test_feature_event_primary_key_with_dayRange(self): + ''' + Verify that primary keys should have expected fields with 'day' field when period is dayRange + ''' + # Reset key properties to default value + FeatureEvents.key_properties = ['feature_id', 'visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + config = {"period": "dayRange"} # set dayRange as a period + expected_primary_keys = ["feature_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", "day"] + + feature_event_stream2 = FeatureEvents(config) # Initialize Events object which sets primary keys + + self.assertEqual(feature_event_stream2.key_properties, expected_primary_keys) From f1523ca5b35f0822001f6895bf6c2fda302b9393 Mon Sep 17 00:00:00 2001 From: savan-chovatiya <80703490+savan-chovatiya@users.noreply.github.com> Date: Thu, 9 Dec 2021 11:42:39 +0530 Subject: [PATCH 4/6] Tdl 15318 pk for track events (#83) * updated PK for track_events stream * updated the comment according to the comments * updated readme file * added unittest * updated unittests, updated config.yml file * run bookmark test for day and hour range * Disabling integration test for merge and enable it in pendo-pk-changes Co-authored-by: harshpatel4_crest Co-authored-by: Harsh <80324346+harshpatel4crest@users.noreply.github.com> --- .circleci/config.yml | 14 ++++---- README.md | 2 +- tap_pendo/streams.py | 5 ++- tests/tap_tester/base.py | 2 +- .../test_track_events_primary_key.py | 34 +++++++++++++++++++ 5 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 tests/unittests/test_track_events_primary_key.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d3d0c72..5f0e805 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ jobs: - store_artifacts: path: htmlcov - add_ssh_keys - - run: - name: 'Integration Tests' - command: | - aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh - source dev_env.sh - source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-pendo tests/tap_tester + # - run: + # name: 'Integration Tests' + # command: | + # aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh + # source dev_env.sh + # source /usr/local/share/virtualenvs/tap-tester/bin/activate + # run-test --tap=tap-pendo tests/tap_tester workflows: version: 2 commit: diff --git a/README.md b/README.md index bf25887..bfacacb 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ This tap: **[track_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` +- Primary key fields: `track_type_id`, `visitor_id`, `account_id`, `server`, `remote_ip`, `user_agent`, `day` or `hour` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` - Transformations: Camel to snake case. diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index 11a26a2..f9400e5 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -693,8 +693,11 @@ def sync(self, state, start_date=None, key_id=None): class TrackEvents(EventsBase): replication_method = "INCREMENTAL" name = "track_events" - key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip'] + key_properties = ['track_type_id', 'visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + def __init__(self, config): + super().__init__(config=config) + self.key_properties.append("day" if self.period == 'dayRange' else "hour") def get_body(self, key_id, period, first): return { diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py index 67ee4f4..6c47bc0 100644 --- a/tests/tap_tester/base.py +++ b/tests/tap_tester/base.py @@ -106,7 +106,7 @@ def expected_metadata(self): self.REPLICATION_KEYS: {'browser_time'} }, "track_events": { - self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, + self.PRIMARY_KEYS: {"track_type_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", event_replication_key}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {event_replication_key} }, diff --git a/tests/unittests/test_track_events_primary_key.py b/tests/unittests/test_track_events_primary_key.py new file mode 100644 index 0000000..3ceeb44 --- /dev/null +++ b/tests/unittests/test_track_events_primary_key.py @@ -0,0 +1,34 @@ +import unittest +from tap_pendo.streams import TrackEvents + +class TestTrackEventsPrimaryKey(unittest.TestCase): + + def test_track_events_primary_key_with_hourRange(self): + ''' + Verify that primary keys should have expected fields with 'hour' field when period is hourRange + ''' + # Reset key properties to default value + TrackEvents.key_properties = ["track_type_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent"] + config = {"period": "hourRange"} # set hourRange as a period + + expected_primary_keys = ["track_type_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", "hour"] + + event_stream1 = TrackEvents(config) # Initialize TrackEvents object which sets primary keys + + # verify if the key properties macthes the expectation + self.assertEqual(event_stream1.key_properties, expected_primary_keys) + + def test_track_events_primary_key_with_dayRange(self): + ''' + Verify that primary keys should have expected fields with 'day' field when period is dayRange + ''' + # Reset key properties to default value + TrackEvents.key_properties = ["track_type_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent"] + config = {"period": "dayRange"} # set dayRange as a period + + expected_primary_keys = ["track_type_id", "visitor_id", "account_id", "server", "remote_ip", "user_agent", "day"] + + event_stream2 = TrackEvents(config) # Initialize TrackEvents object which sets primary keys + + # verify if the key properties macthes the expectation + self.assertEqual(event_stream2.key_properties, expected_primary_keys) From ce2b0cbdbce71b7a64dbbcc8c30cbc94dd2b7cf3 Mon Sep 17 00:00:00 2001 From: savan-chovatiya <80703490+savan-chovatiya@users.noreply.github.com> Date: Thu, 9 Dec 2021 11:49:34 +0530 Subject: [PATCH 5/6] Tdl 15338 fix primary key for events (#84) * TDL-15338: Updated primary keys for events * TDL-15338: Updated integration test * updated readme file * Resolved internal PR review comments * Resolved review comments * run bookmark test with hour and day range Co-authored-by: harshpatel4_crest Co-authored-by: Harsh <80324346+harshpatel4crest@users.noreply.github.com> --- README.md | 2 +- tap_pendo/streams.py | 3 +- tests/tap_tester/base.py | 2 +- tests/unittests/test_events_primary_keys.py | 32 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/test_events_primary_keys.py diff --git a/README.md b/README.md index bfacacb..c9f3da4 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ This tap: **[events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` +- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip`, `user_agent`, `day` or `hour` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `day` or `hour` - Transformations: Camel to snake case. diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index f9400e5..f1a9146 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -584,7 +584,7 @@ def get_body(self, key_id, period, first): class Events(LazyAggregationStream): name = "events" DATE_WINDOW_SIZE = 1 - key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip'] + key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] replication_method = "INCREMENTAL" def __init__(self, config): @@ -592,6 +592,7 @@ def __init__(self, config): self.config = config self.period = config.get('period') self.replication_key = "day" if self.period == 'dayRange' else "hour" + self.key_properties.append(self.replication_key) def sync(self, state, start_date=None, key_id=None): update_currently_syncing(state, self.name) diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py index 6c47bc0..ec288af 100644 --- a/tests/tap_tester/base.py +++ b/tests/tap_tester/base.py @@ -86,7 +86,7 @@ def expected_metadata(self): self.REPLICATION_KEYS: {event_replication_key} }, "events": { - self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"}, + self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip", "user_agent", event_replication_key}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {event_replication_key} }, diff --git a/tests/unittests/test_events_primary_keys.py b/tests/unittests/test_events_primary_keys.py new file mode 100644 index 0000000..4c0db0f --- /dev/null +++ b/tests/unittests/test_events_primary_keys.py @@ -0,0 +1,32 @@ +import unittest +from tap_pendo.streams import Events + +class TestEventsPrimaryKeys(unittest.TestCase): + + def test_event_primary_key_with_hourRange(self): + ''' + Verify that primary keys should have expected fields with 'hour' field when period is hourRange + ''' + # Reset key properties to default value + Events.key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + config = {"period": "hourRange"} # set hourRange as a period + expected_primary_keys = ["visitor_id", "account_id", "server", "remote_ip", "user_agent", "hour"] + + event_stream1 = Events(config) # Initialize Events object which sets primary keys + + self.assertEqual(event_stream1.key_properties, expected_primary_keys) + + + + def test_event_primary_key_with_dayRange(self): + ''' + Verify that primary keys should have expected fields with 'day' field when period is dayRange + ''' + # Reset key properties to default value + Events.key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent'] + config = {"period": "dayRange"} # set dayRange as a period + expected_primary_keys = ["visitor_id", "account_id", "server", "remote_ip", "user_agent", "day"] + + event_stream2 = Events(config) # Initialize Events object which sets primary keys + + self.assertEqual(event_stream2.key_properties, expected_primary_keys) From e1c57e4148d6644ca53cb04584b87c787a69cdb0 Mon Sep 17 00:00:00 2001 From: savan-chovatiya <80703490+savan-chovatiya@users.noreply.github.com> Date: Thu, 9 Dec 2021 12:29:40 +0530 Subject: [PATCH 6/6] Tdl 15148 guide events pk (#85) * updated PK for guide events * added unittests * updated readme file Co-authored-by: harshpatel4_crest --- README.md | 2 +- tap_pendo/schemas/guide_events.json | 6 ++++++ tap_pendo/streams.py | 2 +- tests/tap_tester/base.py | 2 +- .../test_guide_events_primary_key.py | 19 +++++++++++++++++++ 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/test_guide_events_primary_key.py diff --git a/README.md b/README.md index c9f3da4..e9feed3 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ This tap: **[guide_events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)** - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation) -- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip` +- Primary key fields: `guide_id`, `guide_step_id`, `visitor_id`, `type`, `account_id`, `browser_time`, `server_name`, `url` - Replication strategy: INCREMENTAL (query filtered) - Bookmark: `browserTime` - Transformations: Camel to snake case. diff --git a/tap_pendo/schemas/guide_events.json b/tap_pendo/schemas/guide_events.json index 5c4e973..d40b10a 100644 --- a/tap_pendo/schemas/guide_events.json +++ b/tap_pendo/schemas/guide_events.json @@ -31,6 +31,12 @@ "guide_id": { "type": ["null", "string"] }, + "guide_step_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, "latitude": { "type": ["null", "number"] }, diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py index f1a9146..ec4a387 100644 --- a/tap_pendo/streams.py +++ b/tap_pendo/streams.py @@ -726,7 +726,7 @@ def get_body(self, key_id, period, first): class GuideEvents(EventsBase): replication_method = "INCREMENTAL" name = "guide_events" - key_properties = ['visitor_id', 'account_id', 'server_name', 'remote_ip'] + key_properties = ['guide_id', 'guide_step_id', 'visitor_id', 'type', 'account_id', 'browser_time', 'server_name', 'url'] def __init__(self, config): super().__init__(config=config) diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py index ec288af..3937af4 100644 --- a/tests/tap_tester/base.py +++ b/tests/tap_tester/base.py @@ -96,7 +96,7 @@ def expected_metadata(self): self.REPLICATION_KEYS: {event_replication_key} }, "guide_events": { - self.PRIMARY_KEYS: {"visitor_id", "account_id", "server_name", "remote_ip"}, + self.PRIMARY_KEYS: {"guide_id", "guide_step_id", "visitor_id", "type", "account_id", "browser_time", "server_name", "url"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {'browser_time'} }, diff --git a/tests/unittests/test_guide_events_primary_key.py b/tests/unittests/test_guide_events_primary_key.py new file mode 100644 index 0000000..424442f --- /dev/null +++ b/tests/unittests/test_guide_events_primary_key.py @@ -0,0 +1,19 @@ +import unittest +from tap_pendo.streams import GuideEvents + +class TestGuideEventsPrimaryKey(unittest.TestCase): + + def test_guide_event_primary_key(self): + ''' + Verify the primary keys for 'guide_events' stream + ''' + # initialize config + config = {} + # expected primary key + expected_primary_keys = ['guide_id', 'guide_step_id', 'visitor_id', 'type', 'account_id', 'browser_time', 'server_name', 'url'] + + # Initialize GuideEvents object which sets primary keys + guide_events = GuideEvents(config) + + # verify the Primary Key + self.assertEqual(guide_events.key_properties, expected_primary_keys)