Determining the encoding of large OWASP Dependency Check XML reports …

…was very slow. Fixes #803. (#804)

components/collector/src/source_collectors/owasp_dependency_check.py

@@ -8,7 +8,7 @@
 
 from dateutil.parser import isoparse
 
-from collector_utilities.type import Namespaces, Entity, Entities, Response, Responses, Value
+from collector_utilities.type import Namespaces, Entity, Entities, Response, Responses, URL, Value
 from collector_utilities.functions import parse_source_response_xml_with_namespace
 from .source_collector import FileSourceCollector, SourceUpToDatenessCollector
 
@@ -20,6 +20,13 @@ class OWASPDependencyCheckBase(FileSourceCollector, ABC):  # pylint: disable=abs
                          for version in ("2.0", "2.1", "2.2")]
     file_extensions = ["xml"]
 
+    def _get_source_responses(self, api_url: URL) -> Responses:
+        responses = super()._get_source_responses(api_url)
+        for response in responses:
+            if not response.encoding:
+                response.encoding = "utf-8"  # Assume UTF-8, detecting encoding on large XML files is very slow.
+        return responses
+
 
 class OWASPDependencyCheckSecurityWarnings(OWASPDependencyCheckBase):
     """Collector to get security warnings from OWASP Dependency Check."""

components/collector/tests/unittests/source_collectors_tests/source_collector_test_case.py

@@ -24,6 +24,7 @@ def collect(self, metric, *,
                 get_request_json_return_value=None,
                 get_request_json_side_effect=None,
                 get_request_content="",
+                get_request_encoding="",
                 get_request_text="",
                 post_request_side_effect=None,
                 post_request_json_return_value=None,
@@ -34,6 +35,8 @@ def collect(self, metric, *,
             mock_get_request.json.side_effect = get_request_json_side_effect
         else:
             mock_get_request.json.return_value = get_request_json_return_value
+        if get_request_encoding != "":
+            mock_get_request.encoding = get_request_encoding
         mock_get_request.content = get_request_content
         mock_get_request.text = get_request_text
         mock_post_request = Mock()

components/collector/tests/unittests/source_collectors_tests/test_owasp_dependency_check.py

@@ -96,3 +96,17 @@ def test_source_up_to_dateness(self):
         timezone_info = timezone(timedelta(hours=2))
         expected_age = (datetime.now(timezone_info) - datetime(2018, 10, 3, 13, 1, 24, 784, tzinfo=timezone_info)).days
         self.assert_measurement(response, value=str(expected_age))
+
+    def test_source_up_to_dateness_no_encoding(self):
+        """Test that the source age in days is returned, also when the XML has no encoding specified."""
+        xml = """<?xml version="1.0"?>
+        <analysis xmlns="https://jeremylong.github.io/DependencyCheck/dependency-check.2.0.xsd">
+            <projectInfo>
+                <reportDate>2018-10-03T13:01:24.784+0200</reportDate>
+            </projectInfo>
+        </analysis>"""
+        metric = dict(type="source_up_to_dateness", addition="max", sources=self.sources)
+        response = self.collect(metric, get_request_text=xml, get_request_encoding=None)
+        timezone_info = timezone(timedelta(hours=2))
+        expected_age = (datetime.now(timezone_info) - datetime(2018, 10, 3, 13, 1, 24, 784, tzinfo=timezone_info)).days
+        self.assert_measurement(response, value=str(expected_age))

docs/CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 <!-- The line "## <square-bracket>Unreleased</square-bracket>" is replaced by the ci/release.py script with the new release version and release date. -->
 
+## [Unreleased]
+
+### Fixed
+
+- Determining the encoding of large OWASP Dependency Check XML reports was very slow. Fixes [#803](https://github.com/ICTU/quality-time/issues/803).
+
 ## [0.19.0] - [2019-11-17]
 
 ### Fixed