TDL-17610: Multiple apps support and data fetching

README.md

@@ -246,6 +246,7 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during
    - `period` (string, `ABCdef123`): `dayRange` or `hourRange`
    - `lookback_window` (integer): 10 (For event objects. Default: 0)
    - `request_timeout` (integer): 300 (For passing timeout to the request. Default: 300)
+   - `app_ids` (string, `abcd123, Abcd567`): (comma seperated appIDs. If this parameter is not provided, then the data will be collected from all the apps.)
 
    ```json
    {
@@ -254,7 +255,8 @@ Interrupted syncs for Event type stream are resumed via a bookmark placed during
      "period": "dayRange",
      "lookback_window": 10,
      "request_timeout": 300,
-     "include_anonymous_visitors": "true"
+     "include_anonymous_visitors": "true",
+     "app_ids": "abcd123, Abcd567"
    }
    ```
 

tap_pendo/__init__.py

@@ -151,13 +151,34 @@ def sync(config, state, catalog):
         singer.write_state(state)
     LOGGER.info("Finished sync")
 
+# To check that given number is numeric or not
+def isInteger(n):
+    try:
+        int(n)
+        return False
+    except Exception:
+        return True
+
 @utils.handle_top_exception(LOGGER)
 def main():
 
     # Parse command line arguments
     args = utils.parse_args(REQUIRED_CONFIG_KEYS)
 
+    # If app_ids is not in configure file then get all apps data
+    app_ids = args.config.get("app_ids", "expandAppIds(\"*\")").replace(" ", "") or "expandAppIds(\"*\")"
+    # If app_ids is given in configure file then check all app_ids are numeric or not
+    invalid_app_ids = []
+    if app_ids != "expandAppIds(\"*\")":
+        app_ids = app_ids.split(",")
+        invalid_app_ids = list(filter(isInteger, app_ids))
+
+    args.config["app_ids"] = app_ids
+
     if args.discover:
+        # If non numeric app_ids are given in configure file then raise exception
+        if len(invalid_app_ids) > 0:
+            raise Exception('Invalid appIDs provided during the configuration:{}'.format(invalid_app_ids)) from None
         do_discover(args.config)
     elif args.catalog:
         state = args.state

tap_pendo/schemas/features.json

@@ -1,6 +1,9 @@
 {
   "type": ["null", "object"],
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/guides.json

@@ -1,6 +1,9 @@
 {
   "type": ["null", "object"],
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/pages.json

@@ -2,6 +2,9 @@
   "type": ["null", "object"],
   "additional_properties": false,
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/schemas/poll_events.json

@@ -2,6 +2,9 @@
     "additional_properties": false,
     "type": "object",
     "properties": {
+      "app_id": {
+        "type": ["null", "number"]
+      },
       "account_id": {
         "type": ["null", "string"]
       },

tap_pendo/schemas/track_types.json

@@ -2,6 +2,9 @@
   "type": ["null", "object"],
   "additional_properties": false,
   "properties": {
+    "app_id": {
+      "type": ["null", "number"]
+    },
     "created_by_user": {
       "type": ["null", "object"],
       "properties": {

tap_pendo/streams.py

@@ -537,7 +537,7 @@ def get_body(self):
                 "all-features",
                 "pipeline": [{
                     "source": {
-                        "features": None
+                        "features": {"appId": self.config["app_ids"]}
                     }
                 }, {
                     "sort": ["id"]
@@ -676,7 +676,7 @@ def get_body(self, period, first, end):
             "request": {
                 "pipeline": [{
                     "source": {
-                        "events": None,
+                        "events": {"appId": self.config["app_ids"]},
                         "timeSeries": {
                             "period": period,
                             "first": first,
@@ -709,7 +709,7 @@ def get_body(self, period, first):
             "request": {
                 "pipeline": [{
                     "source": {
-                        "pollEvents": None,
+                        "pollEvents": {"appId": self.config["app_ids"]},
                         "timeSeries": {
                             "period": period,
                             "first": first,
@@ -824,7 +824,7 @@ def get_body(self):
                 "name": "all-track-types",
                 "pipeline": [{
                     "source": {
-                        "trackTypes": None
+                        "trackTypes": {"appId": self.config["app_ids"]}
                     }
                 }, {
                     "sort": ["id"]
@@ -849,7 +849,7 @@ def get_body(self):
                 "all-guides",
                 "pipeline": [{
                     "source": {
-                        "guides": None
+                        "guides": {"appId": self.config["app_ids"]}
                     }
                 }, {
                     "sort": ["id"]
@@ -875,7 +875,7 @@ def get_body(self):
                 "all-pages",
                 "pipeline": [{
                     "source": {
-                        "pages": None
+                        "pages": {"appId": self.config["app_ids"]}
                     }
                 }, {
                     "sort": ["id"]

tests/tap_tester/test_all_fields.py

@@ -87,6 +87,12 @@ def test_run(self):
 
                 # verify all fields for each stream are replicated
                 self.assertSetEqual(expected_all_keys, actual_all_keys)
+
+                # Verify we have more than one app data
+                # below four streams are independent of the app_id
+                if stream not in ["accounts", "visitors", "metadata_accounts", "metadata_visitors"]:
+                    records_appid_set = set([message.get('data').get('app_id') for message in messages.get("messages")])
+                    self.assertGreater(len(records_appid_set), 1, msg=f"We have only one app's records for {stream}")
 
 
 

tests/unittests/test_app_ids_configurable.py

@@ -0,0 +1,50 @@
+import unittest
+from unittest import mock
+from argparse import Namespace
+from tap_pendo import main
+
+
+class TestAppIdConfiguration(unittest.TestCase):
+    @mock.patch("tap_pendo.utils.parse_args", side_effect=lambda required_config_keys: Namespace(config={"start_date": "", "x_pendo_integration_key": "", "period":""},  discover=True))
+    @mock.patch("tap_pendo.discover_streams", side_effect=lambda config: {})
+    @mock.patch('tap_pendo.do_discover')
+    def test_app_ids_not_in_config(self, mocked_do_discover, mocked_discover_stream, mocked_parse_args):
+        """
+            To verify that if app_ids is not in configure file then select all apps and run discover mode.
+        """
+        main()
+        config = {'app_ids': 'expandAppIds("*")', 'start_date': '', 'x_pendo_integration_key': '', 'period': ''}
+        mocked_do_discover.assert_called_with(config)
+
+    @mock.patch("tap_pendo.utils.parse_args", side_effect=lambda required_config_keys: Namespace(config={"app_ids": "123, 456","start_date": "", "x_pendo_integration_key": "", "period":""},  discover=True))
+    @mock.patch("tap_pendo.discover_streams", side_effect=lambda config: {})
+    @mock.patch('tap_pendo.do_discover')
+    def test_app_ids_coma_seperated_string_in_config(self, mocked_do_discover, mocked_discover_stream, mocked_parse_args):
+        """
+            To verify that if app_ids is comma seperated string in configure file then get list of those app_ids and run discover mode.
+        """
+        main()
+        config = {'app_ids': ['123', '456'], 'start_date': '', 'x_pendo_integration_key': '', 'period': ''}
+        mocked_do_discover.assert_called_with(config)
+
+    @mock.patch("tap_pendo.utils.parse_args", side_effect=lambda required_config_keys: Namespace(config={"app_ids": "","start_date": "", "x_pendo_integration_key": "", "period":""},  discover=True))
+    @mock.patch("tap_pendo.discover_streams", side_effect=lambda config: {})
+    @mock.patch('tap_pendo.do_discover')
+    def test_app_ids_empty_in_config(self, mocked_do_discover, mocked_discover_stream, mocked_parse_args):
+        """
+            To verify that if app_ids is blank string or empty string in configure file then select all apps and run discover mode.
+        """
+        main()
+        config = {'app_ids': 'expandAppIds("*")', 'start_date': '', 'x_pendo_integration_key': '', 'period': ''}
+        mocked_do_discover.assert_called_with(config)
+
+    @mock.patch("tap_pendo.utils.parse_args", side_effect=lambda required_config_keys: Namespace(config={"app_ids": "123, test, test123, 123test,  ","start_date": "", "x_pendo_integration_key": "", "period":""},  discover=True))
+    def test_app_ids_valid_app_ids_with_invalid_app_ids_config(self, mocked_parse_args):
+        """
+            To verify that if app_ids is comma seperated blanks with string in configure file then then raise exception.
+        """
+
+        with self.assertRaises(Exception) as e:
+            main()
+
+        self.assertEqual(str(e.exception), "Invalid appIDs provided during the configuration:['test', 'test123', '123test', '']", "Not get expected exception")