TDL-17610: Multiple apps support and data fetching

tap_pendo/schemas/features.json

@@ -1,6 +1,9 @@
 {
   "type": ["null", "object"],
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/guides.json

@@ -1,6 +1,9 @@
 {
   "type": ["null", "object"],
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/pages.json

@@ -2,6 +2,9 @@
   "type": ["null", "object"],
   "additional_properties": false,
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/poll_events.json

@@ -2,6 +2,9 @@
     "additional_properties": false,
     "type": "object",
     "properties": {
+      "app_id": {
+        "type": ["null", "number"]
+      },
       "account_id": {
         "type": ["null", "string"]
       },

tap_pendo/schemas/track_types.json

@@ -2,6 +2,9 @@
   "type": ["null", "object"],
   "additional_properties": false,
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/streams.py

@@ -115,6 +115,16 @@ class Stream():
 
     def __init__(self, config=None):
         self.config = config
+
+        # If app_ids are not given in config then select all apps data
+        self.app_ids = self.config.get('app_ids', "expandAppIds(\"*\")").replace(" ", "")
+
+        # If app_ids are empty string or space contains string then select all apps data
+        self.app_ids = self.app_ids or "expandAppIds(\"*\")"
+
+        # If appIds are given then create list of app_ids
+        if self.app_ids != "expandAppIds(\"*\")":
+            self.app_ids = self.app_ids.split(",")
 
     def send_request_get_results(self, req):
         # Set request timeout to config param `request_timeout` value.
@@ -537,7 +547,7 @@ def get_body(self):
                 "all-features",
                 "pipeline": [{
                     "source": {
-                        "features": None
+                        "features": {"appId": self.app_ids}
                     }
                 }, {
                     "sort": ["id"]
@@ -676,7 +686,7 @@ def get_body(self, period, first, end):
             "request": {
                 "pipeline": [{
                     "source": {
-                        "events": None,
+                        "events": {"appId": self.app_ids},
                         "timeSeries": {
                             "period": period,
                             "first": first,
@@ -709,7 +719,7 @@ def get_body(self, period, first):
             "request": {
                 "pipeline": [{
                     "source": {
-                        "pollEvents": None,
+                        "pollEvents": {"appId": self.app_ids},
                         "timeSeries": {
                             "period": period,
                             "first": first,
@@ -824,7 +834,7 @@ def get_body(self):
                 "name": "all-track-types",
                 "pipeline": [{
                     "source": {
-                        "trackTypes": None
+                        "trackTypes": {"appId": self.app_ids}
                     }
                 }, {
                     "sort": ["id"]
@@ -849,7 +859,7 @@ def get_body(self):
                 "all-guides",
                 "pipeline": [{
                     "source": {
-                        "guides": None
+                        "guides": {"appId": self.app_ids}
                     }
                 }, {
                     "sort": ["id"]
@@ -875,7 +885,7 @@ def get_body(self):
                 "all-pages",
                 "pipeline": [{
                     "source": {
-                        "pages": None
+                        "pages": {"appId": self.app_ids}
                     }
                 }, {
                     "sort": ["id"]

tests/tap_tester/test_all_fields.py

@@ -87,6 +87,12 @@ def test_run(self):
 
                 # verify all fields for each stream are replicated
                 self.assertSetEqual(expected_all_keys, actual_all_keys)
+
+                # Verify we have more than one app data
+                # below four streams are independent of the app_id
+                if stream not in ["accounts", "visitors", "metadata_accounts", "metadata_visitors"]:
+                    records_appid_set = set([message.get('data').get('app_id') for message in messages.get("messages")])
+                    self.assertGreater(len(records_appid_set), 1, msg=f"We have only one app's records for {stream}")
 
 
 

tests/unittests/test_app_ids_configurable.py

@@ -0,0 +1,29 @@
+import unittest
+from tap_pendo.streams import Features
+
+
+class TestAppIdConfiguration(unittest.TestCase):
+    def test_app_ids_not_in_config(self):
+        """
+            To verify that if app_ids is not in config than select all apps.
+        """
+        config = {}
+        stream_obj = Features(config)
+        self.assertEqual(stream_obj.app_ids,  "expandAppIds(\"*\")", "Both values are not same")
+
+    def test_app_ids_empty_in_config(self):
+        """
+            To verify that if app_ids is blank string or empty string in config than select all apps.
+        """
+        config = {"app_ids": " "}
+        stream_obj = Features(config)
+        self.assertEqual(stream_obj.app_ids,  "expandAppIds(\"*\")", "Both values are not same")
+
+    def test_app_ids_comaseperated_string_in_config(self):
+        """
+            To verify that f app_ids is comma seperated string in config than get list of those app_ids.
+        """
+        config = {"app_ids": "test1, test2"}
+        stream_obj = Features(config)
+        self.assertEqual(stream_obj.app_ids,  ["test1", "test2"], "Both values are not same")
+