MISP Feed Limit Fix (#36105)

* added fix for the limit parameter

* Posibale solution

* Posibale solution

* Fixed debug log

* Added sort to the result

* pre-commit fixes

* added rn

* Updated README

* store last run only if the time is bigger

* added ut

* Trigger Build

* Added value to the last run

* fixed cr note

* added ut

* cr notes

* improve rn

* updated docker image

* Updated candidate calculation

* pre-commit fixes

* last notes fixes

* pre-commit & cr notes

Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py

@@ -2,11 +2,9 @@
 from CommonServerPython import *  # noqa: F401
 import urllib3
 
-
 # disable insecure warnings
 urllib3.disable_warnings()
 
-
 INDICATOR_TO_GALAXY_RELATION_DICT: Dict[str, Any] = {
     ThreatIntel.ObjectsNames.ATTACK_PATTERN: {
         FeedIndicatorType.File: EntityRelationship.Relationships.INDICATOR_OF,
@@ -216,7 +214,7 @@ def handle_file_type_fields(raw_type: str, indicator_obj: Dict[str, Any]) -> Non
     indicator_obj['fields'][raw_type.upper()] = hash_value
 
 
-def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int, from_timestamp: str | None = None
+def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int, from_timestamp: Optional[int] = None
                       ) -> Dict[str, Any]:
     """
     Creates a dictionary in the format required by MISP to be used as a query.
@@ -237,11 +235,11 @@ def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, pa
         'page': page
     }
     if from_timestamp:
-        params['attribute_timestamp'] = from_timestamp
+        params['attribute_timestamp'] = str(from_timestamp)
     return params
 
 
-def parsing_user_query(query: str, limit: int, page: int = 1, from_timestamp: str | None = None) -> Dict[str, Any]:
+def parsing_user_query(query: str, limit: int, page: int = 1, from_timestamp: Optional[int] | None = None) -> Dict[str, Any]:
     """
     Parsing the query string created by the user by adding necessary argument and removing unnecessary arguments
     Args:
@@ -258,7 +256,7 @@ def parsing_user_query(query: str, limit: int, page: int = 1, from_timestamp: st
         if params.get("timestamp"):
             params['attribute_timestamp'] = params.pop("timestamp")
         if from_timestamp:
-            params['attribute_timestamp'] = from_timestamp
+            params['attribute_timestamp'] = str(from_timestamp)
     except Exception as err:
         demisto.debug(str(err))
         raise DemistoException(f'Could not parse user query. \nError massage: {err}')
@@ -383,7 +381,7 @@ def create_and_add_relationships(indicator_obj: Dict[str, Any], galaxy_indicator
         galaxy_indicator_type = galaxy_indicator['type']
 
         indicator_to_galaxy_relation = INDICATOR_TO_GALAXY_RELATION_DICT[galaxy_indicator_type][indicator_obj_type]
-        galaxy_to_indicator_relation = EntityRelationship.Relationships.\
+        galaxy_to_indicator_relation = EntityRelationship.Relationships. \
             RELATIONSHIPS_NAMES[indicator_to_galaxy_relation]
 
         indicator_relation = EntityRelationship(
@@ -514,6 +512,24 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st
     )
 
 
+def update_candidate(last_run: dict, last_run_timestamp: Optional[int], latest_indicator_timestamp: Optional[int],
+                     latest_indicator_value: str):
+    """
+    Update the candidate timestamp and value based on the latest and last run values.
+
+    Args:
+        last_run: a dictionary containing the last run information, including the timestamp, page, and indicator value.
+        last_run_timestamp: the timestamp of the last run.
+        latest_indicator_timestamp: the timestamp of the latest indicator.
+        latest_indicator_value: the value of the latest indicator.
+    """
+    candidate_timestamp = last_run.get('candidate_timestamp') or last_run_timestamp
+    if (not candidate_timestamp
+            or (latest_indicator_timestamp and latest_indicator_timestamp > candidate_timestamp)):
+        last_run['candidate_timestamp'] = latest_indicator_timestamp
+        last_run['candidate_value'] = latest_indicator_value
+
+
 def fetch_attributes_command(client: Client, params: Dict[str, str]):
     """
     Fetching indicators from the feed to the Indicators tab.
@@ -529,31 +545,55 @@ def fetch_attributes_command(client: Client, params: Dict[str, str]):
     feed_tags = argToList(params.get("feedTags", []))
     attribute_types = argToList(params.get('attribute_types', ''))
     fetch_limit = client.max_indicator_to_fetch
-
+    last_run = demisto.getLastRun()
+    total_fetched_indicators = 0
     query = params.get('query', None)
-    last_run = demisto.getLastRun().get('timestamp') or ""
-    params_dict = parsing_user_query(query, LIMIT, from_timestamp=last_run) if query else\
-        build_params_dict(tags=tags, attribute_type=attribute_types, limit=LIMIT, page=1, from_timestamp=last_run)
+    last_run_timestamp = arg_to_number(last_run.get('last_indicator_timestamp'))
+    last_run_page = last_run.get('page') or 1
+    last_run_value = last_run.get('last_indicator_value') or ""
+    params_dict = parsing_user_query(query, LIMIT, from_timestamp=last_run_timestamp) if query else \
+        build_params_dict(tags=tags, attribute_type=attribute_types, limit=LIMIT,
+                          page=last_run_page, from_timestamp=last_run_timestamp)
+
     search_query_per_page = client.search_query(params_dict)
     demisto.debug(f'params_dict: {params_dict}')
+
     while len(search_query_per_page.get("response", {}).get("Attribute", [])):
         demisto.debug(f'search_query_per_page number of attributes:\
                       {len(search_query_per_page.get("response", {}).get("Attribute", []))} page: {params_dict["page"]}')
+        search_query_per_page.get("response", {}).get("Attribute", []).sort(key=lambda x: x['timestamp'], reverse=False)
         indicators = build_indicators(client, search_query_per_page, attribute_types,
                                       tlp_color, params.get('url'), reputation, feed_tags)
+
+        total_fetched_indicators += len(indicators)
+        latest_indicator = search_query_per_page['response']['Attribute']
+        latest_indicator_timestamp = arg_to_number(latest_indicator[-1]['timestamp'])
+        latest_indicator_value = latest_indicator[-1]['value']
+
+        if last_run_timestamp == latest_indicator_timestamp and latest_indicator_value == last_run_value:
+            # No new indicators since last run, no need to fetch again
+            demisto.debug("No new indicators found since last run")
+            return
+
         for iter_ in batch(indicators, batch_size=2000):
             demisto.createIndicators(iter_)
         params_dict['page'] += 1
-        last_run = search_query_per_page['response']['Attribute'][-1]['timestamp']
+        update_candidate(last_run, last_run_timestamp,
+                         latest_indicator_timestamp, latest_indicator_value)
         # Note: The limit is applied after indicators are created,
         # so the total number of indicators may slightly exceed the limit due to page size constraints.
-        if fetch_limit and fetch_limit <= len(indicators):
-            demisto.debug(f"Reached the limit of indicators to fetch. The number of indicators fetched is: {len(indicators)}")
-            break
+        if fetch_limit and fetch_limit <= total_fetched_indicators:
+            demisto.setLastRun(last_run | {"page": params_dict["page"]})
+            demisto.debug(
+                f"Reached the limit of indicators to fetch."
+                f" The number of indicators fetched is: {total_fetched_indicators}")
+            return
+
         search_query_per_page = client.search_query(params_dict)
     if error_message := search_query_per_page.get('Error'):
         raise DemistoException(f"Error in API call - check the input parameters and the API Key. Error: {error_message}")
-    demisto.setLastRun({'timestamp': last_run})
+    demisto.setLastRun({'last_indicator_timestamp': last_run.get("candidate_timestamp"),
+                        'last_indicator_value': last_run.get("candidate_value")})
 
 
 def main():  # pragma: no cover

Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml

@@ -153,7 +153,7 @@ script:
   script: '-'
   type: python
   subtype: python3
-  dockerimage: demisto/python3:3.11.10.111526
+  dockerimage: demisto/python3:3.11.10.115887
 fromversion: 5.5.0
 tests:
 - MISPfeed Test

Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py

@@ -313,7 +313,8 @@ def test_search_query_indicators_pagination(mocker):
                                          'object_relation': None, 'category': 'Payload delivery',
                                          'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620',
                                          'timestamp': '1607517728', 'distribution': '5', 'sharing_group_id': '0',
-                                         'comment': 'malspam', 'deleted': False, 'disable_correlation': False, 'first_seen': None,
+                                         'comment': 'malspam', 'deleted': False, 'disable_correlation': False,
+                                         'first_seen': None,
                                          'last_seen': None, 'value': 'val2', 'Event': {}}]}}
     returned_result_2 = {'response': {'Attribute': []}}
     mocker.patch.object(Client, '_http_request', side_effect=[returned_result_1, returned_result_2])
@@ -322,6 +323,7 @@ def test_search_query_indicators_pagination(mocker):
         'filters': {'category': ['Payload delivery']},
     }
     mocker.patch("FeedMISP.LIMIT", new=2000)
+    mocker.patch.object(demisto, 'getLastRun', return_value={})
     mocker.patch.object(demisto, 'setLastRun')
     mocker.patch.object(demisto, 'createIndicators')
     fetch_attributes_command(client, params_dict)
@@ -375,3 +377,128 @@ def test_parsing_user_query_timestamp_deprecated():
                  ' "tags": {"OR": ["tlp:%"]}}')
     params = parsing_user_query(query_str, limit=2)
     assert good_query == json.dumps(params)
+
+
+def test_ignore_last_fetched_indicator(mocker):
+    """
+    Given:
+        - The fetch_attributes_command function is called with a client object and a params_dict.
+    When:
+        - The last fetched indicator is returned when already fetched.
+    Then:
+        - The fetch_attributes_command function should ignore the last fetched indicator and continue fetching new indicators.
+    """
+    client = Client(base_url="example",
+                    authorization="auth",
+                    verify=False,
+                    proxy=False,
+                    timeout=60,
+                    performance=False,
+                    max_indicator_to_fetch=2000
+                    )
+    mocked_result = {'response':
+                     {'Attribute': [{'id': '1', 'event_id': '1', 'object_id': '0',
+                                     'object_relation': None, 'category': 'Payload delivery',
+                                     'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620',
+                                     'timestamp': '1607517728', 'distribution': '5', 'sharing_group_id': '0',
+                                     'comment': 'malspam', 'deleted': False, 'disable_correlation': False,
+                                     'first_seen': None, 'last_seen': None,
+                                     'value': 'test', 'Event': {}}]}}
+    mocker.patch.object(Client, '_http_request', side_effect=[mocked_result])
+    params_dict = {
+        'type': 'attribute',
+        'filters': {'category': ['Payload delivery']},
+    }
+    mocked_last_run = {"last_indicator_timestamp": "1607517728", "last_indicator_value": "test"}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mocked_last_run)
+    mocker.patch.object(demisto, 'setLastRun')
+    mocker.patch.object(demisto, 'createIndicators')
+    fetch_attributes_command(client, params_dict)
+    indicators = demisto.createIndicators.call_args
+    assert not indicators  # No indicators should be created since the latest indicator was already fetched
+
+
+def test_fetch_new_indicator_after_last_indicator_been_ignored(mocker):
+    """
+    Given:
+        - The fetch_attributes_command function is called with a client object and a params_dict.
+    When:
+        - The latest retrieved indicators been ignored and new indicator is fetched.
+    Then:
+        - The fetch_attributes_command function should fetch the next indicator and set the new last run.
+    """
+    client = Client(base_url="example",
+                    authorization="auth",
+                    verify=False,
+                    proxy=False,
+                    timeout=60,
+                    performance=False,
+                    max_indicator_to_fetch=2000
+                    )
+    mocked_result_1 = {'response':
+                       {'Attribute': [{'id': '1', 'event_id': '1', 'object_id': '0',
+                                       'object_relation': None, 'category': 'Payload delivery',
+                                       'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620',
+                                       'timestamp': '1607517728', 'distribution': '5', 'sharing_group_id': '0',
+                                       'comment': 'malspam', 'deleted': False, 'disable_correlation': False,
+                                       'first_seen': None, 'last_seen': None,
+                                       'value': 'test1', 'Event': {}},
+                                      {'id': '2', 'event_id': '2', 'object_id': '0',
+                                       'object_relation': None, 'category': 'Payload delivery',
+                                       'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620',
+                                       'timestamp': '1607517729', 'distribution': '5', 'sharing_group_id': '0',
+                                       'comment': 'malspam', 'deleted': False, 'disable_correlation': False,
+                                       'first_seen': None,
+                                       'last_seen': None, 'value': 'test2', 'Event': {}}]}}
+    mocked_result_2 = {'response':
+                       {'Attribute': []}}
+    mocker.patch.object(Client, '_http_request', side_effect=[mocked_result_1, mocked_result_2])
+    params_dict = {
+        'type': 'attribute',
+        'filters': {'category': ['Payload delivery']},
+    }
+    mocked_last_run = {"last_indicator_timestamp": "1607517728", "last_indicator_value": "test1"}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mocked_last_run)
+    setLastRun_mocked = mocker.patch.object(demisto, 'setLastRun')
+    mocker.patch.object(demisto, 'createIndicators')
+    fetch_attributes_command(client, params_dict)
+    indicators = demisto.createIndicators.call_args[0][0]
+    # The last ignored indicator will be re-fetched as we query his timestamp,
+    # but the new last run will be updated with the new indicator.
+    assert len(indicators) == 2
+    assert setLastRun_mocked.called
+
+
+def test_set_last_run_pagination(mocker):
+    """
+    Given:
+         - The set_last_run_pagination function is called with a list of indicators, a next_page value, and a last_run dictionary.
+    When:
+        - The function is called to set the last run with the appropriate values.
+    Then:
+        - Ensure the last run is set correctly with the appropriate values
+    """
+    from FeedMISP import update_candidate
+
+    # Sample indicators
+    indicators = [
+        {'value': 'test1', 'timestamp': '1607517728'},
+        {'value': 'test2', 'timestamp': '1607517729'}
+    ]
+
+    # Test parameters
+    last_run = {"last_indicator_timestamp": "1607517727", "last_indicator_value": "test0"}
+    last_run_timestamp = last_run["last_indicator_timestamp"]
+    last_run_value = last_run["last_indicator_value"]
+    latest_indicator_timestamp = indicators[-1]["timestamp"]
+    latest_indicator_value = indicators[-1]["value"]
+
+    # Call the function
+    update_candidate(last_run, last_run_timestamp,
+                     latest_indicator_timestamp, latest_indicator_value)
+
+    # Assert that setLastRun was called with the correct arguments
+    expected_last_run = {'last_indicator_timestamp': last_run_timestamp, 'candidate_timestamp': latest_indicator_timestamp,
+                         'last_indicator_value': last_run_value,
+                         'candidate_value': latest_indicator_value}
+    assert last_run == expected_last_run

Packs/FeedMISP/Integrations/FeedMISP/README.md

@@ -30,6 +30,7 @@ To ingest feeds via a URL, you could use one of the following content packs:
     | Source Reliability | Reliability of the source providing the intelligence data. | True |
     | Feed Fetch Interval |  | False |
     | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False |
+    | Max. indicators per fetch | Limit the number of indicators retrieved in a fetch run. | False |
     | MISP Attribute Tags | Attributes having one of the tags, or being an attribute of an event having one of the tags, will be returned. You can enter a comma-separated list of tags, for example <tag1,tag2,tag3>. The list of MISP tags can be found in your MISP instance under 'Event Actions'>'List Tags' | False |
     | MISP Attribute Types | Attributes of one of these types will be returned. You can enter a comma-separated list of types, for example <type1,type2,type3>. The list of MISP types can be found in your MISP instance then 'Event Actions'>'Search Attributes'>'Type dropdown list' | False |
     | Query | JSON query to filter MISP attributes. When the query parameter is used, Attribute Types and Attribute Tags parameters are not used. You can check for the correct syntax at https://&lt;Your MISP url&gt;/servers/openapi\#operation/restSearchAttributes | False |

Packs/FeedMISP/ReleaseNotes/1_0_38.md

@@ -0,0 +1,8 @@
+
+#### Integrations
+
+##### MISP Feed
+
+- Fixed an issue where the ***fetch-incidcators*** command reached a docker timeout.
+- Improved implementation of the ***fetch-incidcators*** command to ensure the feed completes successfully when no new indicators are available and that indicators are pulled only once.
+- Updated the Docker image to: *demisto/python3:3.11.10.115887*.

Packs/FeedMISP/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "MISP Feed",
     "description": "Indicators feed from MISP",
     "support": "xsoar",
-    "currentVersion": "1.0.37",
+    "currentVersion": "1.0.38",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",