Migrate to the new SonarQube issue structure.

Migrate to the new SonarQube issue structure introduced in SonarQube 10.2. This means:
- Remove the "Severities" parameter.
- Add an "Impact severities" parameter. The possible values of the impact severity parameter are "low", "medium", and "high".
- Migrate the severity parameter values to the new values according to the mapping documented by SonarSource: https://docs.sonarsource.com/sonarqube/latest/user-guide/issues/#severity-mapping.
- Drop the issue type parameter
- Add a "software quality" parameter and use the impactSoftwareQualities parameter for the api/issues/search endpoint
- Add a "clean code attribute" parameter and use the cleanCodeAttributeCategories parameter for the api/issues/search endpoint

TODO:
- Change the security_types parameter so it does not depend on the issue type.
- Document that the lowest supported SonarQube version is 10.2.

Closes #8354.

components/api_server/src/initialization/migrations.py

@@ -20,6 +20,7 @@ def perform_migrations(database: Database) -> None:
                 change_accessibility_violation_metrics_to_violations(report),
                 fix_branch_parameters_without_value(report),
                 change_ci_subject_types_to_development_environment(report),
+                change_sonarqube_parameters(report),
             ]
         ):
             change_description = " and to ".join([change for change in changes if change])
@@ -120,6 +121,43 @@ def add_source_parameter_hash_to_latest_measurement(database: Database, report)
     return count
 
 
+def change_sonarqube_parameters(report) -> str:
+    """Replace the SonarQube parameters to adapt to the new (SonarQube 10.2) issue structure.
+
+    Return a description of the change, if any.
+    """
+    # Added after Quality-time v5.13.0, see https://github.com/ICTU/quality-time/issues/8354
+    change = ""
+    # Severity mapping conform https://docs.sonarsource.com/sonarqube/latest/user-guide/issues/#severity-mapping:
+    severity_mapping = {"blocker": "high", "critical": "high", "major": "medium", "minor": "low", "info": "low"}
+    for source in sources(
+        report,
+        metric_types=("security_warnings", "suppressed_violations", "violations"),
+        source_type="sonarqube",
+        parameter="severities",
+    ):
+        old_severities = source["parameters"]["severities"]
+        new_severities = {severity_mapping[severity] for severity in old_severities if severity in severity_mapping}
+        if new_severities:
+            source["parameters"]["impact_severities"] = list(new_severities)
+        del source["parameters"]["severities"]
+        change = "change the SonarQube parameters"
+    for source in sources(
+        report, metric_types=("suppressed_violations", "violations"), source_type="sonarqube", parameter="types"
+    ):
+        del source["parameters"]["types"]
+        change = "change the SonarQube parameters"
+    return change
+
+
+def sources(report, metric_types: Sequence[str], source_type: str, parameter: str):
+    """Yield the sources in the report, filtered by metric type and source type."""
+    for metric in metrics(report, metric_types):
+        for source in metric.get("sources", {}).values():
+            if source["type"] == source_type and parameter in source["parameters"]:
+                yield source
+
+
 def metrics(report, metric_types: Sequence[str] | None = None):
     """Yield the metrics in the report, optionally filtered by metric type."""
     for subject in subjects(report):

components/api_server/tests/initialization/test_migrations.py

@@ -222,3 +222,91 @@ def test_ci_environment_with_title_and_subtitle(self):
         perform_migrations(self.database)
         inserted_report = self.inserted_report(subject_name="CI", subject_description="My CI")
         self.database.reports.replace_one.assert_called_once_with({"_id": "id"}, inserted_report)
+
+
+class SonarQubeParameterTest(MigrationTestCase):
+    """Unit tests for the SonarQube parameter database migration."""
+
+    def existing_report(
+        self,
+        metric_type: str = "violations",
+        sources: dict[SourceId, dict[str, str | dict[str, str | list[str]]]] | None = None,
+    ):
+        """Extend to add sources."""
+        report = super().existing_report(metric_type=metric_type)
+        report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["sources"] = sources
+        return report
+
+    def test_report_without_severity_or_types_parameter(self):
+        """Test that the migration succeeds when the SonarQube source has no severity or types parameter."""
+        self.database.reports.find.return_value = [
+            self.existing_report(sources={SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main"}}})
+        ]
+        perform_migrations(self.database)
+        self.database.reports.replace_one.assert_not_called()
+
+    def test_report_with_violation_metric_but_no_sonarqube(self):
+        """Test that the migration succeeds when a violations metric has no SonarQube sources."""
+        self.database.reports.find.return_value = [self.existing_report(sources={SOURCE_ID: {"type": "sarif"}})]
+        perform_migrations(self.database)
+        self.database.reports.replace_one.assert_not_called()
+
+    def test_report_with_severity_parameter(self):
+        """Test that the migration succeeds when the SonarQube source has a severity parameter."""
+        self.database.reports.find.return_value = [
+            self.existing_report(
+                sources={SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "severities": ["info"]}}},
+            )
+        ]
+        perform_migrations(self.database)
+        inserted_sources = {
+            SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "impact_severities": ["low"]}}
+        }
+        inserted_report = self.inserted_report(sources=inserted_sources)
+        self.database.reports.replace_one.assert_called_once_with({"_id": "id"}, inserted_report)
+
+    def test_report_with_multiple_old_severity_values_that_map_to_the_same_new_value(self):
+        """Test a severity parameter with multiple old values that map to the same new value."""
+        self.database.reports.find.return_value = [
+            self.existing_report(
+                sources={
+                    SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "severities": ["info", "minor"]}},
+                },
+            )
+        ]
+        perform_migrations(self.database)
+        inserted_sources = {
+            SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "impact_severities": ["low"]}}
+        }
+        inserted_report = self.inserted_report(sources=inserted_sources)
+        self.database.reports.replace_one.assert_called_once_with({"_id": "id"}, inserted_report)
+
+    def test_report_with_unknown_old_severity_values(self):
+        """Test that unknown severity parameter values are ignored."""
+        self.database.reports.find.return_value = [
+            self.existing_report(
+                sources={
+                    SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "severities": ["info", ""]}},
+                    SOURCE_ID2: {"type": "sonarqube", "parameters": {"branch": "main", "severities": ["foo"]}},
+                },
+            )
+        ]
+        perform_migrations(self.database)
+        inserted_sources = {
+            SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "impact_severities": ["low"]}},
+            SOURCE_ID2: {"type": "sonarqube", "parameters": {"branch": "main"}},
+        }
+        inserted_report = self.inserted_report(sources=inserted_sources)
+        self.database.reports.replace_one.assert_called_once_with({"_id": "id"}, inserted_report)
+
+    def test_report_with_types_parameter(self):
+        """Test that the migration succeeds when the SonarQube source has a types parameter."""
+        self.database.reports.find.return_value = [
+            self.existing_report(
+                sources={SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main", "types": ["bug"]}}},
+            )
+        ]
+        perform_migrations(self.database)
+        inserted_sources = {SOURCE_ID: {"type": "sonarqube", "parameters": {"branch": "main"}}}
+        inserted_report = self.inserted_report(sources=inserted_sources)
+        self.database.reports.replace_one.assert_called_once_with({"_id": "id"}, inserted_report)

components/collector/src/source_collectors/sonarqube/security_warnings.py

@@ -11,11 +11,9 @@
 class SonarQubeSecurityWarnings(SonarQubeViolations):
     """SonarQube security warnings. The security warnings are a sum of the vulnerabilities and security hotspots."""
 
-    types_parameter = "security_types"
-
     async def _landing_url(self, responses: SourceResponses) -> URL:
         """Extend to return the correct landing url depending on the selected security types."""
-        security_types = self._parameter(self.types_parameter)
+        security_types = self._parameter("security_types")
         base_landing_url = await SourceCollector._landing_url(self, responses)  # noqa: SLF001
         component = self._parameter("component")
         branch = self._parameter("branch")
@@ -24,11 +22,9 @@ async def _landing_url(self, responses: SourceResponses) -> URL:
             landing_path = "dashboard"
         elif "vulnerability" in security_types:
             landing_path = "project/issues"
-            # We don't use self._query_parameter() for the types parameter because when we get here,
-            # the value of the types parameter is fixed
             extra_url_parameters = (
-                f"{self._query_parameter('severities', uppercase=True)}&resolved=false&types=VULNERABILITY"
-                f"{self._query_parameter('tags')}"
+                f"{self._query_parameter('impact_severities', uppercase=True)}&resolved=false&"
+                f"impactSoftwareQualities=SECURITY{self._query_parameter('tags')}"
             )
         else:
             landing_path = "project/security_hotspots"
@@ -37,25 +33,31 @@ async def _landing_url(self, responses: SourceResponses) -> URL:
     async def _get_source_responses(self, *urls: URL) -> SourceResponses:
         """Extend to add urls for the selected security types."""
         api_urls = []
-        security_types = self._parameter(self.types_parameter)
+        security_types = self._parameter("security_types")
         component = self._parameter("component")
         branch = self._parameter("branch")
         base_url = await SonarQubeCollector._api_url(self)  # noqa: SLF001
         if "vulnerability" in security_types:
             api_urls.append(
                 URL(
                     f"{base_url}/api/issues/search?componentKeys={component}&resolved=false&ps=500"
-                    f"{self._query_parameter('severities', uppercase=True)}&branch={branch}&types=VULNERABILITY"
-                    f"{self._query_parameter('tags')}",
+                    f"{self._query_parameter('impact_severities', uppercase=True)}&branch={branch}&"
+                    f"impactSoftwareQualities=SECURITY{self._query_parameter('tags')}",
                 ),
             )
         if "security_hotspot" in security_types:
             api_urls.append(URL(f"{base_url}/api/hotspots/search?projectKey={component}&branch={branch}&ps=500"))
         return await super()._get_source_responses(*api_urls)
 
+    async def _entity(self, issue) -> Entity:
+        """Extend to set the entity type."""
+        entity = await super()._entity(issue)
+        entity["type"] = "vulnerability"
+        return entity
+
     async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
         """Override to parse the selected security types."""
-        security_types = self._parameter(self.types_parameter)
+        security_types = self._parameter("security_types")
         vulnerabilities = (
             await super()._parse_source_responses(SourceResponses(responses=[responses[0]]))
             if "vulnerability" in security_types

components/collector/src/source_collectors/sonarqube/suppressed_violations.py

@@ -35,8 +35,7 @@ async def _get_source_responses(self, *urls: URL) -> SourceResponses:
         all_issues_api_url = URL(f"{url}/api/issues/search?componentKeys={component}&branch={branch}")
         resolved_issues_api_url = URL(
             f"{all_issues_api_url}&statuses=RESOLVED&resolutions=WONTFIX,FALSE-POSITIVE&additionalFields=comments"
-            f"{self._query_parameter('severities', uppercase=True)}"
-            f"{self._query_parameter(self.types_parameter, uppercase=True)}&ps=500",
+            f"{self._query_parameter('impact_severities', uppercase=True)}&ps=500",
         )
         return await super()._get_source_responses(*[*urls, resolved_issues_api_url, all_issues_api_url])
 

components/collector/src/source_collectors/sonarqube/violations.py

@@ -14,21 +14,14 @@ class SonarQubeViolations(SonarQubeCollector):
     """SonarQube violations metric. Also base class for metrics that measure specific rules."""
 
     rules_configuration = ""  # Subclass responsibility
-    types_parameter = "types"
 
     async def _landing_url(self, responses: SourceResponses) -> URL:
         """Extend to add the issues path and parameters."""
         url = await super()._landing_url(responses)
         component = self._parameter("component")
         branch = self._parameter("branch")
         landing_url = f"{url}/project/issues?id={component}&branch={branch}&resolved=false"
-        return URL(
-            landing_url
-            + self._query_parameter("severities", uppercase=True)
-            + self._query_parameter("tags")
-            + self._query_parameter(self.types_parameter, uppercase=True)
-            + self.__rules_url_parameter(),
-        )
+        return URL(landing_url + self.__url_parameters())
 
     async def _api_url(self) -> URL:
         """Extend to add the issue search path and parameters."""
@@ -38,20 +31,30 @@ async def _api_url(self) -> URL:
         # If there's more than 500 issues only the first 500 are returned. This is no problem since we limit
         # the number of "entities" sent to the server anyway (that limit is 100 currently).
         api = f"{url}/api/issues/search?componentKeys={component}&branch={branch}&resolved=false&ps=500"
-        return URL(
-            api
-            + self._query_parameter("severities", uppercase=True)
+        return URL(api + self.__url_parameters())
+
+    def __url_parameters(self) -> str:
+        """Return the parameters common to the API URL and the landing URL."""
+        return (
+            self._query_parameter("impact_severities", uppercase=True)
+            + self._query_parameter("impacted_software_qualities", uppercase=True)
+            + self._query_parameter("clean_code_attribute_categories", uppercase=True)
             + self._query_parameter("tags")
-            + self._query_parameter(self.types_parameter, uppercase=True)
-            + self.__rules_url_parameter(),
+            + self.__rules_url_parameter()
         )
 
     def _query_parameter(self, parameter_key: str, uppercase: bool = False) -> str:
         """Return the multiple choice parameter as query parameter that can be passed to SonarQube."""
         parameter_value = ",".join(sorted(cast(list[str], self._parameter(parameter_key))))
         if uppercase:
             parameter_value = parameter_value.upper()
-        return "" if parameter_value == self.__default_value(parameter_key) else f"&{parameter_key}={parameter_value}"
+        parameter_to_query_parameter_mapping = {
+            "impact_severities": "impactSeverities",
+            "impacted_software_qualities": "impactSoftwareQualities",
+            "clean_code_attribute_categories": "cleanCodeAttributeCategories",
+        }
+        query_parameter = parameter_to_query_parameter_mapping.get(parameter_key, parameter_key)
+        return "" if parameter_value == self.__default_value(parameter_key) else f"&{query_parameter}={parameter_value}"
 
     def __rules_url_parameter(self) -> str:
         """Return the rules url parameter, if any."""
@@ -81,12 +84,15 @@ async def __issue_landing_url(self, issue_key: str) -> URL:
 
     async def _entity(self, issue) -> Entity:
         """Create an entity from an issue."""
+        impacts = [
+            f"{impact['severity']} at {impact['softwareQuality']}".lower() for impact in issue.get("impacts", [])
+        ]
         return Entity(
             key=issue["key"],
             url=await self.__issue_landing_url(issue["key"]),
             message=issue["message"],
-            severity=issue.get("severity", "no severity").lower(),
-            type=issue["type"].lower(),
+            impacts=", ".join(impacts),
+            clean_code_attribute_category=issue["cleanCodeAttributeCategory"].lower(),
             component=issue["component"],
             creation_date=issue["creationDate"],
             update_date=issue["updateDate"],

components/collector/tests/source_collectors/sonarqube/base.py

@@ -23,9 +23,10 @@ def setUp(self):
     def entity(  # noqa: PLR0913
         key: str,
         component: str,
-        entity_type: str,
         message: str,
-        severity: str | None = None,
+        entity_type: str | None = None,
+        impacts: str | None = None,
+        clean_code_attribute_category: str | None = None,
         resolution: str | None = None,
         rationale: str | None = None,
         review_priority: str | None = None,
@@ -43,14 +44,17 @@ def entity(  # noqa: PLR0913
         entity = Entity(
             key=key,
             component=component,
-            type=entity_type,
             message=message,
             url=url,
             creation_date=creation_date,
             update_date=update_date,
         )
-        if severity is not None:
-            entity["severity"] = severity
+        if entity_type:
+            entity["type"] = entity_type
+        if impacts is not None:
+            entity["impacts"] = impacts
+        if clean_code_attribute_category is not None:
+            entity["clean_code_attribute_category"] = clean_code_attribute_category
         if resolution is not None:
             entity["resolution"] = resolution
         if rationale is not None:

components/collector/tests/source_collectors/sonarqube/test_security_warnings.py

@@ -14,8 +14,10 @@ class SonarQubeSecurityWarningsTest(SonarQubeTestCase):
     DASHBOARD_URL = f"{SONARQUBE_URL}/dashboard?id=id{BRANCH}"
     HOTSPOTS_API = f"{API_URL}/hotspots/search?projectKey=id{BRANCH}&ps=500"
     HOTSPOTS_LANDING_URL = f"{LANDING_URL}/security_hotspots?id=id{BRANCH}"
-    VULNERABILITIES_API = f"{API_URL}/issues/search?componentKeys=id&resolved=false&ps=500{BRANCH}&types=VULNERABILITY"
-    VULNERABILITIES_LANDING_URL = f"{LANDING_URL}/issues?id=id{BRANCH}&resolved=false&types=VULNERABILITY"
+    VULNERABILITIES_API = (
+        f"{API_URL}/issues/search?componentKeys=id&resolved=false&ps=500{BRANCH}&impactSoftwareQualities=SECURITY"
+    )
+    VULNERABILITIES_LANDING_URL = f"{LANDING_URL}/issues?id=id{BRANCH}&resolved=false&impactSoftwareQualities=SECURITY"
 
     def setUp(self):
         """Extend to set up SonarQube security warnings."""
@@ -27,8 +29,8 @@ def setUp(self):
                     "key": "vulnerability1",
                     "message": "message1",
                     "component": "component1",
-                    "severity": "INFO",
-                    "type": "VULNERABILITY",
+                    "impacts": [{"severity": "low", "softwareQuality": "security"}],
+                    "cleanCodeAttributeCategory": "RESPONSIBLE",
                     "creationDate": "2020-08-30T22:48:52+0200",
                     "updateDate": "2020-09-30T22:48:52+0200",
                     "tags": ["bug"],
@@ -37,8 +39,8 @@ def setUp(self):
                     "key": "vulnerability2",
                     "message": "message2",
                     "component": "component2",
-                    "severity": "MAJOR",
-                    "type": "VULNERABILITY",
+                    "impacts": [{"severity": "medium", "softwareQuality": "security"}],
+                    "cleanCodeAttributeCategory": "CONSISTENT",
                     "creationDate": "2019-08-30T22:48:52+0200",
                     "updateDate": "2019-09-30T22:48:52+0200",
                     "tags": ["bug", "other tag"],
@@ -97,7 +99,8 @@ def setUp(self):
                 component="component1",
                 entity_type="vulnerability",
                 message="message1",
-                severity="info",
+                impacts="low at security",
+                clean_code_attribute_category="responsible",
                 creation_date="2020-08-30T22:48:52+0200",
                 update_date="2020-09-30T22:48:52+0200",
                 tags="bug",
@@ -106,7 +109,8 @@ def setUp(self):
                 key="vulnerability2",
                 component="component2",
                 entity_type="vulnerability",
-                severity="major",
+                impacts="medium at security",
+                clean_code_attribute_category="consistent",
                 creation_date="2019-08-30T22:48:52+0200",
                 update_date="2019-09-30T22:48:52+0200",
                 message="message2",