Merge pull request #297 from aviaconstructor/improved_date_formats

Implemented a better date conversion for 'date:' and 'modified:' fields.

sigma/rule.py

@@ -5,6 +5,7 @@
 from enum import Enum, auto
 from datetime import date, datetime
 import yaml
+import re
 import sigma
 from sigma.types import SigmaType, SigmaNull, SigmaString, SigmaNumber, sigma_type
 from sigma.modifiers import (
@@ -847,6 +848,45 @@ class instantiation of an object derived from the SigmaRuleBase class and the er
         SigmaRule object. Else the first recognized error is raised as exception.
         """
         errors = []
+
+        def get_rule_as_date(name: str, exception_class) -> Optional[date]:
+            """
+            Accepted string based date formats are in range 1000-01-01 .. 3999-12-31:
+              * XXXX-XX-XX                                 -- fully corresponds to yaml date format
+              * XXXX/XX/XX, XXXX/XX/X, XXXX/X/XX, XXXX/X/X -- often occurs in the US-based sigmas
+            Not accepted are ambiguous dates such as:
+                2024-01-1, 24-1-24, 24/1/1, ...
+            """
+            nonlocal errors, rule, source
+            result = rule.get(name)
+            if (
+                result is not None
+                and not isinstance(result, date)
+                and not isinstance(result, datetime)
+            ):
+                error = True
+                try:
+                    result = str(result)  # forcifully convert whatever the type is into string
+                    accepted_regexps = (
+                        "([1-3][0-9][0-9][0-9])-([01][0-9])-([0-3][0-9])",  # 1000-01-01 .. 3999-12-31
+                        "([1-3][0-9][0-9][0-9])/([01]?[0-9])/([0-3]?[0-9])",  # 1000/1/1, 1000/01/01 .. 3999/12/31
+                    )
+                    for date_regexp in accepted_regexps:
+                        matcher = re.fullmatch(date_regexp, result)
+                        if matcher:
+                            result = date(int(matcher[1]), int(matcher[2]), int(matcher[3]))
+                            error = False
+                            break
+                except Exception:
+                    pass
+                if error:
+                    errors.append(
+                        exception_class(
+                            f"Rule {name} '{ result }' is invalid, use yyyy-mm-dd", source=source
+                        )
+                    )
+            return result
+
         # Rule identifier may be empty or must be valid UUID
         rule_id = rule.get("id")
         if rule_id is not None:
@@ -944,32 +984,10 @@ class instantiation of an object derived from the SigmaRuleBase class and the er
                     )
 
         # parse rule date if existing
-        rule_date = rule.get("date")
-        if rule_date is not None:
-            if not isinstance(rule_date, date) and not isinstance(rule_date, datetime):
-                try:
-                    rule_date = date(*(int(i) for i in rule_date.split("-")))
-                except ValueError:
-                    errors.append(
-                        sigma_exceptions.SigmaDateError(
-                            f"Rule date '{ rule_date }' is invalid, must be yyyy-mm-dd",
-                            source=source,
-                        )
-                    )
+        rule_date = get_rule_as_date("date", sigma_exceptions.SigmaDateError)
 
         # parse rule modified if existing
-        rule_modified = rule.get("modified")
-        if rule_modified is not None:
-            if not isinstance(rule_modified, date) and not isinstance(rule_modified, datetime):
-                try:
-                    rule_modified = date(*(int(i) for i in rule_modified.split("-")))
-                except ValueError:
-                    errors.append(
-                        sigma_exceptions.SigmaModifiedError(
-                            f"Rule modified '{ rule_modified }' is invalid, must be yyyy-mm-dd",
-                            source=source,
-                        )
-                    )
+        rule_modified = get_rule_as_date("modified", sigma_exceptions.SigmaModifiedError)
 
         # Rule fields validation
         rule_fields = rule.get("fields")

tests/test_rule.py

@@ -879,15 +879,68 @@ def test_sigmarule_bad_status_type():
 
 
 def test_sigmarule_bad_date():
-    with pytest.raises(sigma_exceptions.SigmaDateError, match="Rule date.*test.yml"):
-        SigmaRule.from_dict({"date": "bad"}, source=sigma_exceptions.SigmaRuleLocation("test.yml"))
+    """This test uses string data type as date representation in yaml"""
+    bad_string_dates = (
+        "bad",
+        " 2024-11-24",
+        "2024-11-24 ",
+        "2024 11-24",
+        "24-11-24",
+        "02-02-02",
+        "4000-01-01",
+        "10000-01-01",
+        "2022-01/01",
+        "2022/01-01",
+    )
+    for test_string in bad_string_dates:
+        match_string = f"Rule date '{test_string}' is invalid, use yyyy-mm-dd"
+        with pytest.raises(sigma_exceptions.SigmaDateError, match=match_string) as ex:
+            SigmaRule.from_yaml(
+                f"""
+                title: Test
+                date: '{test_string}'  # try a string
+                logsource:
+                    product: foobar
+                detection:
+                    selection_1:
+                        fieldA: valueA
+                    condition: selection_1
+                """
+            )
+            assert False, f"Did not throw SigmaDateError on date {test_string}"
 
 
 def test_sigmarule_bad_modified():
-    with pytest.raises(sigma_exceptions.SigmaModifiedError, match="Rule modified.*test.yml"):
-        SigmaRule.from_dict(
-            {"modified": "bad"}, source=sigma_exceptions.SigmaRuleLocation("test.yml")
-        )
+    """
+    This test uses yaml ability to recognize dates.
+    Therefore, here 4000-01-01 will be interpreted as a correct yaml date.
+    """
+    bad_dates = (
+        "bad",
+        "24-11-24",
+        "02-02-02",
+        "2024-5-5",
+        "10000-01-01",
+        "2022-01/01",
+        "2022/01-01",
+        "2022 01/01",
+    )
+    for test_string in bad_dates:
+        match_string = f"Rule modified '{test_string}' is invalid, use yyyy-mm-dd"
+        with pytest.raises(sigma_exceptions.SigmaModifiedError, match=match_string) as ex:
+            SigmaRule.from_yaml(
+                f"""
+                title: Test
+                modified: {test_string}  # this can be recognized as date by yaml parser
+                logsource:
+                    product: foobar
+                detection:
+                    selection_1:
+                        fieldA: valueA
+                    condition: selection_1
+                """
+            )
+            assert False, f"Did not throw SigmaModifiedError on date {test_string}"
 
 
 def test_sigmarule_bad_falsepositives():
@@ -932,25 +985,36 @@ def test_sigmarule_date():
 
 
 def test_modified_date():
-    expected_date = date(3000, 11, 3)
-    rule = SigmaRule.from_yaml(
-        """
-    title: Test
-    id: cafedead-beef-0000-1111-0123456789ab
-    level: medium
-    status: test
-    date: 3000-01-02
-    modified: 3000-11-03
-    logsource:
-        product: foobar
-    detection:
-        selection_1:
-            fieldA: valueA
-        condition: selection_1
-    """
-    )
-    assert rule is not None
-    assert rule.modified == expected_date
+    validDates = {
+        "3000-12-31": date(3000, 12, 31),  # can appear as a generic date in the future
+        "2999-04-04": date(2999, 4, 4),
+        "2024-11-22": date(2024, 11, 22),
+        "1970-01-01": date(1970, 1, 1),  # can appear as a generic date in the past
+        "1900-12-31": date(1900, 12, 31),  # not much useful, but correct
+        "2024/11/22": date(2024, 11, 22),  # US-based sigmas have such dates
+        "2024/1/2": date(2024, 1, 2),
+        "1970/1/1": date(1970, 1, 1),
+    }
+    for test_string, expected_date in validDates.items():
+        rule = SigmaRule.from_yaml(
+            f"""
+            title: Test
+            id: cafedead-beef-0000-1111-0123456789ab
+            level: medium
+            status: test
+            date: {test_string}       # possibly a yaml date
+            modified: '{test_string}' # always a string data type converted into date
+            logsource:
+                product: foobar
+            detection:
+                selection_1:
+                    fieldA: valueA
+                condition: selection_1
+            """
+        )
+        assert rule is not None
+        assert rule.date == expected_date, f"bad 'date' for '{test_string}'"
+        assert rule.modified == expected_date, f"bad 'modified' for '{test_string}'"
 
 
 def test_sigmarule_datetime():