[CommonServerPython] Add offset support to lookback

Packs/Base/ReleaseNotes/1_32_39.md

@@ -0,0 +1,7 @@
+
+#### Scripts
+
+##### CommonServerPython
+
+- Added support for *offset* when fetching incidents with *lookback*.
+- Updated the Docker image to: *demisto/python:2.7.18.76921*.

Packs/Base/Scripts/CommonServerPython/CommonServerPython.py

@@ -10587,6 +10587,7 @@ def get_fetch_run_time_range(last_run, first_fetch, look_back=0, timezone=0, dat
     :rtype: ``Tuple``
     """
     last_run_time = last_run and 'time' in last_run and last_run['time']
+    offset = last_run.get('offset')
     now = get_current_time(timezone)
     if not last_run_time:
         last_run_time = dateparser.parse(first_fetch, settings={'TIMEZONE': 'UTC', 'RETURN_AS_TIMEZONE_AWARE': True})
@@ -10595,7 +10596,7 @@ def get_fetch_run_time_range(last_run, first_fetch, look_back=0, timezone=0, dat
     else:
         last_run_time = dateparser.parse(last_run_time, settings={'TIMEZONE': 'UTC', 'RETURN_AS_TIMEZONE_AWARE': True})
 
-    if look_back and look_back > 0:
+    if not offset and look_back and look_back > 0:
         if now - last_run_time < timedelta(minutes=look_back):
             last_run_time = now - timedelta(minutes=look_back)
 
@@ -10621,6 +10622,26 @@ def get_current_time(time_zone=0):
         demisto.debug('pytz is missing, will not return timeaware object.')
         return now
 
+def calculate_new_offset(old_offset, num_incidents, total_incidents):
+    """ This calculates the new offset based on the response
+
+    :type old_offset: ``int``
+    :param old_offset: The offset from the previous run
+
+    :type num_incidents: ``int``
+    :param num_incidents: The number of incidents returned by the API.
+
+    :type total_incidents: ``int``
+    :param total_incidents: The total number of incidents returned by the API.
+
+    :return: The new offset for the next run.
+    :rtype: ``int``
+    """
+    if not num_incidents:
+        return 0
+    if total_incidents and num_incidents + old_offset >= total_incidents:
+        return 0
+    return old_offset + num_incidents
 
 def filter_incidents_by_duplicates_and_limit(incidents_res, last_run, fetch_limit, id_field):
     """
@@ -10760,7 +10781,7 @@ def get_found_incident_ids(last_run, incidents, look_back, id_field, remove_inci
 
 
 def create_updated_last_run_object(last_run, incidents, fetch_limit, look_back, start_fetch_time, end_fetch_time,
-                                   created_time_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False):
+                                   created_time_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False, new_offset=None):
     """
     Calculates the next fetch time and limit depending the incidents result and creates an updated LastRun object
     with the new time and limit.
@@ -10791,18 +10812,26 @@ def create_updated_last_run_object(last_run, incidents, fetch_limit, look_back,
 
     :type increase_last_run_time: ``bool``
     :param increase_last_run_time: Whether to increase the last run time with one millisecond
+
+    :type new_offset: ``int | None``
+    :param new_offset: The new offset to set in the last run
 
     :return: The new LastRun object
     :rtype: ``Dict``
     """
     demisto.debug("lb: Create updated last run object, len(incidents) is {}," \
-                  "look_back is {}, fetch_limit is {}".format(len(incidents), look_back, fetch_limit))
+                  "look_back is {}, fetch_limit is {}, new_offset is {}".format(len(incidents), look_back, fetch_limit, new_offset))
     remove_incident_ids = True
     new_limit = len(last_run.get('found_incident_ids', [])) + len(incidents) + fetch_limit
-    if len(incidents) == 0:
+    if new_offset:
+        # if we need to update the offset, we need to keep the old time and just update the offset
+        new_last_run = {
+            'time': last_run.get("time"),
+        } 
+    elif len(incidents) == 0:
         new_last_run = {
             'time': end_fetch_time,
-            'limit': fetch_limit
+            'limit': fetch_limit,
         }
     else:        
         latest_incident_fetched_time = get_latest_incident_created_time(incidents, created_time_field, date_format,
@@ -10814,16 +10843,18 @@ def create_updated_last_run_object(last_run, incidents, fetch_limit, look_back,
         if latest_incident_fetched_time == start_fetch_time:
             # we are still on the same time, no need to remove old incident ids
             remove_incident_ids = False
-
-
+
+    if new_offset is not None:
+        new_last_run['offset'] = new_offset
+        new_last_run['limit'] = fetch_limit
     demisto.debug("lb: The new_last_run is: {}, the remove_incident_ids is: {}".format(new_last_run,
                                                                                        remove_incident_ids))
 
     return new_last_run, remove_incident_ids
 
 
 def update_last_run_object(last_run, incidents, fetch_limit, start_fetch_time, end_fetch_time, look_back,
-                           created_time_field, id_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False):
+                           created_time_field, id_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False, new_offset=None):
     """
     Updates the LastRun object with the next fetch time and limit and with the new fetched incident IDs.
 
@@ -10856,6 +10887,10 @@ def update_last_run_object(last_run, incidents, fetch_limit, start_fetch_time, e
 
     :type increase_last_run_time: ``bool``
     :param increase_last_run_time: Whether to increase the last run time with one millisecond
+
+    :type new_offset: ``int | None``
+    :param new_offset: The new offset to set in the last run
+
 
     :return: The updated LastRun object
     :rtype: ``Dict``
@@ -10872,6 +10907,7 @@ def update_last_run_object(last_run, incidents, fetch_limit, start_fetch_time, e
                                                                            created_time_field,
                                                                            date_format,
                                                                            increase_last_run_time,
+                                                                           new_offset
                                                                            )
 
     found_incidents = get_found_incident_ids(last_run, incidents, look_back, id_field, remove_incident_ids)

Packs/Base/Scripts/CommonServerPython/CommonServerPython.yml

@@ -23,4 +23,4 @@ tests:
 - Test-debug-mode
 - Test-CreateDBotScore-With-Reliability
 fromversion: 5.0.0
-dockerimage: demisto/python:2.7.18.63476
+dockerimage: demisto/python:2.7.18.76921

Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py

@@ -8250,6 +8250,97 @@ def test_update_last_run_object(self, args1, expected_results1, args2, expected_
         for id_ in results.get('found_incident_ids').keys():
             assert id_ in expected_results3.get('found_incident_ids')
 
+    @freeze_time("2022-04-07T10:13:00")
+    def test_lookback_with_offset_time_range(self):
+        """
+        Given:
+            A last run with an offset
+
+        When:
+            Calling get_fetch_run_time_range
+
+        Then:
+            The last run should be unchanged, even if there is a lookback.
+        """
+        from CommonServerPython import get_fetch_run_time_range
+        last_time = "2022-04-07T10:13:00"
+        last_run = {"time": last_time, "offset": 3}
+        start_time, end_time = get_fetch_run_time_range(last_run, None, look_back=1)
+        # make sure that the start time is unchanged because of the offset
+        assert start_time == last_time
+        last_run = {"time": last_time, "offset": 0}
+        start_time, end_time = get_fetch_run_time_range(last_run, None, look_back=1)
+        # now the offset is 0, so the look_back should act
+        assert start_time == "2022-04-07T10:12:00"
+
+    def test_lookback_with_offset_update_last_run(self):
+        """
+        Given:
+            A last run
+
+        When:
+            Calling create_updated_last_run_object with a new offset to change
+
+        Then:
+            - The last run is updated with the new offset, and the start time remains as it was.
+            - When the offset needs to be reset, the last time is the latest incident time and the offset resets
+        """
+        from CommonServerPython import create_updated_last_run_object
+        last_time = "2022-04-07T10:13:00"
+        last_run = {"time": last_time, "offset": 3}
+        new_offset = 4
+        new_last_run, _ = create_updated_last_run_object(last_run,
+                                                         self.INCIDENTS,
+                                                         fetch_limit=3,
+                                                         look_back=1,
+                                                         start_fetch_time=last_time,
+                                                         end_fetch_time=datetime.now().isoformat(),
+                                                         created_time_field="created",
+                                                         new_offset=new_offset,
+                                                         )
+        # make sure that the start time is unchanged because of the offset, and the offset is updated
+        assert new_last_run["offset"] == 4
+        assert new_last_run["time"] == last_time
+
+        last_run = {"time": last_time, "offset": new_offset}
+        new_offset = 0
+        new_last_run, _ = create_updated_last_run_object(last_run,
+                                                         self.INCIDENTS,
+                                                         fetch_limit=3,
+                                                         look_back=1,
+                                                         start_fetch_time=last_time,
+                                                         end_fetch_time=datetime.now().isoformat(),
+                                                         created_time_field="created",
+                                                         new_offset=new_offset,
+                                                         )
+        assert new_last_run["offset"] == 0
+        assert new_last_run["time"] == "2022-04-01T10:51:00"
+
+    def test_calculate_new_offset(self):
+        """
+        Test that the new offset for the next run calculated correctly based on the old offset, number of incidents and total number of incidents.
+        The first argument is the old offset, the second is number of incidents and the third is the total number of incidents returned.
+        Given:
+            old offset, number of incidents, total number of incidents (could be None)
+
+        When:
+            Calculating a new offset to the next run
+
+        Then:
+            Make sure that the new offset is correct
+        """
+        from CommonServerPython import calculate_new_offset
+        assert calculate_new_offset(0, 2, 4) == 2
+        assert calculate_new_offset(0, 2, 2) == 0
+        assert calculate_new_offset(0, 2, 3) == 2
+        assert calculate_new_offset(1, 2, 4) == 3
+        assert calculate_new_offset(1, 2, 3) == 0
+        assert calculate_new_offset(1, 2, None) == 3
+
+
+
+
+
 
 class TestTracebackLineNumberAdgustment:
     @staticmethod

Packs/Base/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Base",
     "description": "The base pack for Cortex XSOAR.",
     "support": "xsoar",
-    "currentVersion": "1.32.38",
+    "currentVersion": "1.32.39",
     "author": "Cortex XSOAR",
     "serverMinVersion": "6.0.0",
     "url": "https://www.paloaltonetworks.com/cortex",