Fix SARIF-formatter severity levels (#3824)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Sorin Sbarnea <[email protected]>

src/ansiblelint/formatters/__init__.py

@@ -14,6 +14,7 @@
 
 if TYPE_CHECKING:
     from ansiblelint.errors import MatchError
+    from ansiblelint.rules import BaseRule  # type: ignore[attr-defined]
 
 T = TypeVar("T", bound="BaseFormatter")  # type: ignore[type-arg]
 
@@ -264,7 +265,7 @@ def _to_sarif_rule(self, match: MatchError) -> dict[str, Any]:
                 "text": str(match.message),
             },
             "defaultConfiguration": {
-                "level": self._to_sarif_level(match),
+                "level": self.get_sarif_rule_severity_level(match.rule),
             },
             "help": {
                 "text": str(match.rule.description),
@@ -285,7 +286,7 @@ def _to_sarif_result(self, match: MatchError) -> dict[str, Any]:
 
         result: dict[str, Any] = {
             "ruleId": match.tag,
-            "level": match.level,
+            "level": self.get_sarif_result_severity_level(match),
             "message": {
                 "text": str(match.details)
                 if str(match.details)
@@ -312,6 +313,37 @@ def _to_sarif_result(self, match: MatchError) -> dict[str, Any]:
         return result
 
     @staticmethod
-    def _to_sarif_level(match: MatchError) -> str:
-        # sarif accepts only 4 levels: error, warning, note, none
-        return match.level
+    def get_sarif_rule_severity_level(rule: BaseRule) -> str:
+        """General SARIF severity level for a rule.
+
+        Note: Can differ from an actual result/match severity.
+        Possible values: "none", "note", "warning", "error"
+
+        see: https://github.com/oasis-tcs/sarif-spec/blob/123e95847b13fbdd4cbe2120fa5e33355d4a042b/Schemata/sarif-schema-2.1.0.json#L1934-L1939
+        """
+        if rule.severity in ["VERY_HIGH", "HIGH"]:
+            return "error"
+
+        if rule.severity in ["MEDIUM", "LOW", "VERY_LOW"]:
+            return "warning"
+
+        if rule.severity == "INFO":
+            return "note"
+
+        return "none"
+
+    @staticmethod
+    def get_sarif_result_severity_level(match: MatchError) -> str:
+        """SARIF severity level for an actual result/match.
+
+        Possible values: "none", "note", "warning", "error"
+
+        see: https://github.com/oasis-tcs/sarif-spec/blob/123e95847b13fbdd4cbe2120fa5e33355d4a042b/Schemata/sarif-schema-2.1.0.json#L2066-L2071
+        """
+        if not match.level:
+            return "none"
+
+        if match.level in ["warning", "error"]:
+            return match.level
+
+        return "note"

test/test_formatter_sarif.py

@@ -19,43 +19,61 @@
 class TestSarifFormatter:
     """Unit test for SarifFormatter."""
 
-    rule = AnsibleLintRule()
+    rule1 = AnsibleLintRule()
+    rule2 = AnsibleLintRule()
     matches: list[MatchError] = []
     formatter: SarifFormatter | None = None
     collection = RulesCollection()
+    collection.register(rule1)
+    collection.register(rule2)
 
     def setup_class(self) -> None:
         """Set up few MatchError objects."""
-        self.rule = AnsibleLintRule()
-        self.rule.id = "TCF0001"
-        self.rule.severity = "VERY_HIGH"
-        self.rule.description = "This is the rule description."
-        self.rule.link = "https://rules/help#TCF0001"
-        self.rule.tags = ["tag1", "tag2"]
-        self.collection.register(self.rule)
-
-        self.matches = []
-        self.matches.append(
-            MatchError(
-                message="message",
-                lineno=1,
-                column=10,
-                details="details",
-                lintable=Lintable("filename.yml", content=""),
-                rule=self.rule,
-                tag="yaml[test]",
-            ),
-        )
-        self.matches.append(
-            MatchError(
-                message="message",
-                lineno=2,
-                details="",
-                lintable=Lintable("filename.yml", content=""),
-                rule=self.rule,
-                tag="yaml[test]",
-            ),
+        self.rule1.id = "TCF0001"
+        self.rule1.severity = "VERY_HIGH"
+        self.rule1.description = "This is the rule description."
+        self.rule1.link = "https://rules/help#TCF0001"
+        self.rule1.tags = ["tag1", "tag2"]
+
+        self.rule2.id = "TCF0002"
+        self.rule2.severity = "MEDIUM"
+        self.rule2.link = "https://rules/help#TCF0002"
+        self.rule2.tags = ["tag3", "tag4"]
+
+        self.matches.extend(
+            [
+                MatchError(
+                    message="message1",
+                    lineno=1,
+                    column=10,
+                    details="details1",
+                    lintable=Lintable("filename1.yml", content=""),
+                    rule=self.rule1,
+                    tag="yaml[test1]",
+                    ignored=False,
+                ),
+                MatchError(
+                    message="message2",
+                    lineno=2,
+                    details="",
+                    lintable=Lintable("filename2.yml", content=""),
+                    rule=self.rule1,
+                    tag="yaml[test2]",
+                    ignored=True,
+                ),
+                MatchError(
+                    message="message3",
+                    lineno=666,
+                    column=667,
+                    details="details3",
+                    lintable=Lintable("filename3.yml", content=""),
+                    rule=self.rule2,
+                    tag="yaml[test3]",
+                    ignored=False,
+                ),
+            ],
         )
+
         self.formatter = SarifFormatter(pathlib.Path.cwd(), display_relative_path=True)
 
     def test_format_list(self) -> None:
@@ -81,7 +99,7 @@ def test_sarif_format(self) -> None:
         """Test if the return SARIF object contains the expected results."""
         assert isinstance(self.formatter, SarifFormatter)
         sarif = json.loads(self.formatter.format_result(self.matches))
-        assert len(sarif["runs"][0]["results"]) == 2
+        assert len(sarif["runs"][0]["results"]) == 3
         for result in sarif["runs"][0]["results"]:
             # Ensure all reported entries have a level
             assert "level" in result
@@ -98,16 +116,18 @@ def test_validate_sarif_schema(self) -> None:
         assert driver["name"] == SarifFormatter.TOOL_NAME
         assert driver["informationUri"] == SarifFormatter.TOOL_URL
         rules = driver["rules"]
-        assert len(rules) == 1
+        assert len(rules) == 3
         assert rules[0]["id"] == self.matches[0].tag
         assert rules[0]["name"] == self.matches[0].tag
         assert rules[0]["shortDescription"]["text"] == self.matches[0].message
-        assert rules[0]["defaultConfiguration"]["level"] == "error"
+        assert rules[0]["defaultConfiguration"][
+            "level"
+        ] == SarifFormatter.get_sarif_rule_severity_level(self.matches[0].rule)
         assert rules[0]["help"]["text"] == self.matches[0].rule.description
         assert rules[0]["properties"]["tags"] == self.matches[0].rule.tags
         assert rules[0]["helpUri"] == self.matches[0].rule.url
         results = sarif["runs"][0]["results"]
-        assert len(results) == 2
+        assert len(results) == 3
         for i, result in enumerate(results):
             assert result["ruleId"] == self.matches[i].tag
             assert (
@@ -134,6 +154,9 @@ def test_validate_sarif_schema(self) -> None:
                     "startColumn"
                     not in result["locations"][0]["physicalLocation"]["region"]
                 )
+            assert result["level"] == SarifFormatter.get_sarif_result_severity_level(
+                self.matches[i],
+            )
         assert sarif["runs"][0]["originalUriBaseIds"][SarifFormatter.BASE_URI_ID]["uri"]
         assert results[0]["message"]["text"] == self.matches[0].details
         assert results[1]["message"]["text"] == self.matches[1].message