Tdl 15338 fix primary key for events (#84)

* TDL-15338: Updated primary keys for events * TDL-15338: Updated integration test * updated readme file * Resolved internal PR review comments * Resolved review comments * run bookmark test with hour and day range Co-authored-by: harshpatel4_crest <[email protected]> Co-authored-by: Harsh <[email protected]>
singer-io · Dec 9, 2021 · ce2b0cb · ce2b0cb
1 parent f1523ca
commit ce2b0cb
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -103,7 +103,7 @@ This tap:
 **[events](https://developers.pendo.io/docs/?bash#get-an-account-by-id)**
 
 - Endpoint: [https://api/v1/aggregation](https://app.pendo.io/api/v1/aggregation)
-- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip`
+- Primary key fields: `visitor_id`, `account_id`, `server`, `remote_ip`, `user_agent`, `day` or `hour`
 - Replication strategy: INCREMENTAL (query filtered)
   - Bookmark: `day` or `hour`
 - Transformations: Camel to snake case.

diff --git a/tap_pendo/streams.py b/tap_pendo/streams.py
@@ -584,14 +584,15 @@ def get_body(self, key_id, period, first):
 class Events(LazyAggregationStream):
     name = "events"
     DATE_WINDOW_SIZE = 1
-    key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip']
+    key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent']
     replication_method = "INCREMENTAL"
 
     def __init__(self, config):
         super().__init__(config=config)
         self.config = config
         self.period = config.get('period')
         self.replication_key = "day" if self.period == 'dayRange' else "hour"
+        self.key_properties.append(self.replication_key)
 
     def sync(self, state, start_date=None, key_id=None):
         update_currently_syncing(state, self.name)

diff --git a/tests/tap_tester/base.py b/tests/tap_tester/base.py
@@ -86,7 +86,7 @@ def expected_metadata(self):
                 self.REPLICATION_KEYS: {event_replication_key}
             },
             "events": {
-                self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip"},
+                self.PRIMARY_KEYS: {"visitor_id", "account_id", "server", "remote_ip", "user_agent", event_replication_key},
                 self.REPLICATION_METHOD: self.INCREMENTAL,
                 self.REPLICATION_KEYS: {event_replication_key}
             },

diff --git a/tests/unittests/test_events_primary_keys.py b/tests/unittests/test_events_primary_keys.py
@@ -0,0 +1,32 @@
+import unittest
+from tap_pendo.streams import Events
+
+class TestEventsPrimaryKeys(unittest.TestCase):
+
+    def test_event_primary_key_with_hourRange(self):
+        '''
+            Verify that primary keys should have expected fields with 'hour' field when period is hourRange 
+        '''
+        # Reset key properties to default value
+        Events.key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent']
+        config = {"period": "hourRange"} # set hourRange as a period
+        expected_primary_keys = ["visitor_id", "account_id", "server", "remote_ip", "user_agent", "hour"]
+
+        event_stream1 = Events(config) # Initialize Events object which sets primary keys
+
+        self.assertEqual(event_stream1.key_properties, expected_primary_keys)
+
+
+
+    def test_event_primary_key_with_dayRange(self):
+        '''
+            Verify that primary keys should have expected fields with 'day' field when period is dayRange 
+        '''
+        # Reset key properties to default value
+        Events.key_properties = ['visitor_id', 'account_id', 'server', 'remote_ip', 'user_agent']
+        config = {"period": "dayRange"} # set dayRange as a period
+        expected_primary_keys = ["visitor_id", "account_id", "server", "remote_ip", "user_agent", "day"]
+
+        event_stream2 = Events(config) # Initialize Events object which sets primary keys
+
+        self.assertEqual(event_stream2.key_properties, expected_primary_keys)