add duplicate check for stream-rules

.vscode/settings.json

@@ -1,9 +1,10 @@
 {
     "python.testing.pytestArgs": [
-        "tests"
+      "tests"
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
+    "python.envFile": "${workspaceFolder}/.env",
     "python.analysis.extraPaths": [
       "/mnt/c/Projects"
     ]

README.md

@@ -44,4 +44,4 @@ Docs:
 
 Testing single module with a stub: `python plugins/modules/graylog_stream.py tests/stubs/graylog_streams.json`
 
-Unit-Testing single module: `python -m pytest -r a --fulltrace --color yes tests/units/plugins/modules/test_graylog_stream.py`
+Unit-Testing single module: `python -m pytest -r a --fulltrace --color yes tests/units/plugins/module_utils/test_stream.py`

plugins/module_utils/streams.py

@@ -61,6 +61,14 @@ def rules(self, value) -> None:
     self._rules = value
 
 
+  def equals(self, stream: "StreamBase") -> bool:    
+    return (
+      self.properties_are_equal(stream)
+      and self.started_is_equal(stream)
+      and self.rules_are_equal(stream)
+    )
+
+
   def properties_are_equal(self, stream: "StreamBase") -> bool:
     return (
       self.title == stream.title
@@ -74,31 +82,31 @@ def started_is_equal(self, stream: "StreamBase") -> bool:
 
 
   def rules_are_equal(self, stream: "StreamBase") -> bool:
-    if len(self.rules) != len(stream.rules):
-      return False
-
-    for item in self.rules:
-      if (any(x for x in stream.rules if self._rule_equals(x, item) is False)):
-        return False
-
-    return True
+    add, delete = self.get_rules_changes(stream)
+    return len(add) == 0 and len(delete) == 0
 
 
   # returns tuple(add, delete) lists
-  def get_rules_changes(self, stream: "StreamBase") -> Tuple[list, list]:
-    rules = stream.rules
-    add = []
-    delete = []
+  def get_rules_changes(self, stream_params: "StreamBase") -> Tuple[list, list]:   
+    add_list = []
+    delete_list = []
 
     for item in self.rules:
-      if len(rules) == 0 or any(x for x in rules if self._rule_equals(x, item)) is False:
-        delete.append(item)
-
-    for item in rules:
-      if len(self.rules) == 0 or any(x for x in self.rules if self._rule_equals(x, item) is False):
-        add.append(item)
-
-    return add, delete
+      if len(stream_params.rules) == 0 or any(x for x in stream_params.rules if self._rule_equals(x, item)) is False:
+        delete_items = [x for x in self.rules if self._rule_equals(x, item)]
+        for delete_item in delete_items:
+          if (any(x for x in delete_list if x == delete_item) is False):
+            delete_list.append(delete_item)
+
+    for item in stream_params.rules:
+      existing_items = [x for x in self.rules if self._rule_equals(x, item)]
+      if (len(existing_items) == 0):
+        add_list.append(item)
+      elif (len(existing_items) > 1):
+        for existing_item in existing_items[1:]:
+          delete_list.append(existing_item)
+
+    return add_list, delete_list
 
 
   def _rule_equals(self, a: dict, b: dict) -> bool:

plugins/modules/graylog_stream.py

@@ -170,7 +170,7 @@ def create_stream(module: AnsibleModule, stream_params: StreamParams) -> bool:
   response_stream = json.loads(to_text(response.read(), errors='surrogate_or_strict'))
   stream_id = response_stream['stream_id']
 
-  if module.params['started'] == True:
+  if stream_params.started == True:
     resume_stream(module, stream_id)
 
   return True
@@ -194,11 +194,7 @@ def should_update_stream(state: str, stream: Stream, stream_params: StreamParams
   if state == "present" and stream is None:
     return False
 
-  return (
-    stream.properties_are_equal(stream_params) is False
-    or stream.started_is_equal(stream_params) is False
-    or stream.rules_are_equal(stream_params) is False
-  )
+  return stream.equals(stream_params) is False
 
 
 def update_stream(module: AnsibleModule, stream: Stream, stream_params: StreamParams) -> None:

tests/conftest.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
-sys.path.insert(0,os.path.dirname(os.path.realpath(__file__)) + "../")
+sys.path.insert(0, "/mnt/c/Projects")

tests/units/plugins/module_utils/test_stream.py

@@ -52,8 +52,10 @@ def test_properties_are_equal_compares_index_set_id(self, a: str, b: str, expect
 
 
   @pytest.mark.parametrize("a,b,expected", [ 
+    ([], [ { "field": "foo" } ], False),
+    ([ { "field": "foo" } ], [], False),
     ([ { "field": "foo" } ], [ { "field": "foo" } ], True),
-    ([ { "field": "foo" } ], [ { "field": "bar" } ], False),
+    ([ { "field": "foo" } ], [ { "field": "bar" } ], False)
   ])
   def test_rules_are_equal(self, a: list, b: list, expected: bool):
     stream_a = StreamBase()
@@ -126,3 +128,47 @@ def test_get_rules_changes_returns_correct_lists_a(self):
 
     assert 0 == len(delete)
     assert 1 == len(add)
+
+
+  def test_get_rules_changes_removes_duplicates(self):
+    stream = StreamBase()
+    stream.rules = [
+      {
+        'id': 'abc0',
+        'field': 'foo',
+        'value': '1',          
+        'type': 1,
+      },
+      {
+        'id': 'abc1',
+        'field': 'bar',
+        'value': 'baz',          
+        'type': 1,
+      },
+      {
+        'id': 'abc2',
+        'field': 'bar',
+        'value': 'baz',
+        'type': 1,
+      }
+    ]
+
+    stream_params = StreamBase()
+    stream_params.rules = [
+      {
+        'field': 'foo',
+        'value': '1',
+        'type': 1,
+      },
+      {
+        'id': 'abc1',
+        'field': 'bar',
+        'value': 'baz',          
+        'type': 1,
+      },
+    ]
+
+    add, delete = stream.get_rules_changes(stream_params)
+
+    assert sum(1 for x in delete if x['field'] == 'bar') == 1
+    assert 0 == len(add)