From 49b51c8f82aa564df49cc6147c458bb42361f185 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 28 Nov 2022 16:18:43 -0800 Subject: [PATCH] Update and clean importer, replace existing tests with 3 new tests #971 Reference: https://github.com/nexB/vulnerablecode/issues/971 Signed-off-by: John M. Horan --- vulnerabilities/importers/apache_httpd.py | 116 ++--------- vulnerabilities/tests/conftest.py | 2 +- vulnerabilities/tests/test_apache_httpd.py | 194 +++++------------- ...y-CVE-1999-1199-apache-httpd-expected.json | 34 +++ ...-CVE-2021-44224-apache-httpd-expected.json | 34 +++ 5 files changed, 136 insertions(+), 244 deletions(-) create mode 100644 vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-1999-1199-apache-httpd-expected.json create mode 100644 vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-2021-44224-apache-httpd-expected.json diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index f7702cd1c..adfbd39cc 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -8,16 +8,14 @@ # import asyncio -import datetime import urllib -import dateparser import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from univers.version_constraint import VersionConstraint from univers.version_range import GenericVersionRange from univers.versions import SemverVersion -from univers.version_constraint import VersionConstraint from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage @@ -27,13 +25,12 @@ from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.severity_systems import APACHE_HTTPD -# from vulnerabilities.utils import nearest_patched_package - class ApacheHTTPDImporter(Importer): base_url = "https://httpd.apache.org/security/json/" spdx_license_expression = "Apache-2.0" + license_url = "https://www.apache.org/licenses/" # For now, don't use the GH API # def set_api(self): @@ -55,7 +52,6 @@ def advisory_data(self): yield self.to_advisory(data) def to_advisory(self, data): - # cve = data["CVE_data_meta"]["ID"] alias = data["CVE_data_meta"]["ID"] descriptions = data["description"]["description_data"] description = None @@ -78,9 +74,7 @@ def to_advisory(self, data): ) break reference = Reference( - # reference_id=cve, reference_id=alias, - # url=urllib.parse.urljoin(self.base_url, f"{cve}.json"), url=urllib.parse.urljoin(self.base_url, f"{alias}.json"), severities=severities, ) @@ -91,72 +85,11 @@ def to_advisory(self, data): for version_data in products["version"]["version_data"]: versions_data.append(version_data) - # print("\n\n==> versions_data = {}\n".format(versions_data)) - for version in versions_data: - # print("\n\tversion = {}\n".format(version)) - import json - - # print("\n\tversion = \n{}\n".format(json.dumps(version, indent=2))) - affected_version_range = self.to_version_ranges(versions_data) - # fixed_version = [] - # date_published = "" - - # for entry in data["timeline"]: - # value = entry["value"] - # # if "released" in entry["value"]: - # if "released" in value: - # # fixed_version.append(entry["value"]) - # fixed_version.append(value.split(" ")[0]) - # date_published = get_published_date(entry["time"]) - - # affected_packages = [] - # fixed_packages = [] - - # for version in versions_data: - # affected_package = AffectedPackage( - # package=PackageURL( - # type="generic", - # name="apache_httpd", - # ), - # # affected_version_range=affected_version_range, - # affected_version_range=version.get("version_value", "ERROR!!"), - # fixed_version=fixed_version[0], - # # fixed_version="to come", - # ) - # affected_packages.append(affected_package) - - # for version_range in fixed_version_ranges: - # fixed_packages.extend( - # [ - # PackageURL(type="apache", name="httpd", version=version) - # for version in self.version_api.get("apache/httpd").valid_versions - # if SemverVersion(version) in version_range - # ] - # ) - - # for version_range in affected_version_ranges: - # affected_packages.extend( - # [ - # PackageURL(type="apache", name="httpd", version=version) - # for version in self.version_api.get("apache/httpd").valid_versions - # if SemverVersion(version) in version_range - # ] - # ) - return AdvisoryData( - # vulnerability_id=cve, aliases=[alias], summary=description, - # affected_packages=nearest_patched_package(affected_packages, fixed_packages), - # affected_packages=AffectedPackage( - # package=PackageURL( - # type="apache", - # name="httpd", - # ), - # affected_version_range=affected_version_range - # ) affected_packages=[ AffectedPackage( package=PackageURL( @@ -174,23 +107,30 @@ def to_version_ranges(self, versions_data): for version_data in versions_data: version_value = version_data["version_value"] range_expression = version_data["version_affected"] + if range_expression == ">=" or range_expression == "!<": - constraints.append(VersionConstraint( - comparator=">=", - version=SemverVersion(version_value), - )) + constraints.append( + VersionConstraint( + comparator=">=", + version=SemverVersion(version_value), + ) + ) if range_expression == "<=": - constraints.append(VersionConstraint( - comparator="<=", - version=SemverVersion(version_value), - )) + constraints.append( + VersionConstraint( + comparator="<=", + version=SemverVersion(version_value), + ) + ) if range_expression == "=" or range_expression == "?=": - constraints.append(VersionConstraint( - comparator="=", - version=SemverVersion(version_value), - )) + constraints.append( + VersionConstraint( + comparator="=", + version=SemverVersion(version_value), + ) + ) return GenericVersionRange(constraints=constraints) @@ -207,20 +147,6 @@ def fetch_links(url): return links -# From osv.py -# def get_published_date(raw_data): -# published = raw_data.get("published") -# return published and dateparser.parse(date_string=published) - - -def get_published_date(published): - # return published and dateparser.parse(date_string=published) - # above gives result like this: "date_published": "2021-12-20T00:00:00" - # so does this: - published = datetime.datetime.strptime(published, "%Y-%m-%d") - return published - - ignore_tags = { "AGB_BEFORE_AAA_CHANGES", "APACHE_1_2b1", diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index f771d0571..ff5fae3b0 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -25,7 +25,7 @@ def no_rmtree(monkeypatch): # Step 2: Run test for importer only if it is activated (pytestmark = pytest.mark.skipif(...)) # Step 3: Migrate all the tests collect_ignore = [ - # "test_apache_httpd.py", + "test_apache_httpd.py", "test_apache_kafka.py", "test_apache_tomcat.py", "test_api.py", diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 60362c224..0ed8c763d 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -11,167 +11,65 @@ import os from unittest import TestCase -from packageurl import PackageURL -from univers.version_range import VersionRange +from univers.version_constraint import VersionConstraint +from univers.version_range import GenericVersionRange +from univers.versions import SemverVersion -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter - -# from vulnerabilities.importers.apache_httpd import to_advisory from vulnerabilities.package_managers import GitHubTagsAPI -from vulnerabilities.package_managers import PackageVersion from vulnerabilities.tests import util_tests -from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -# TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_httpd", "CVE-1999-1199.json") TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_httpd") -# class TestApacheHTTPDImporter(TestCase): -# @classmethod -# def setUpClass(cls): -# data_source_cfg = {"etags": {}} -# cls.data_src = ApacheHTTPDImporter(1, config=data_source_cfg) -# known_versions = [PackageVersion("1.3.2"), PackageVersion("1.3.1"), PackageVersion("1.3.0")] -# cls.data_src.version_api = GitHubTagsAPI(cache={"apache/httpd": known_versions}) -# with open(TEST_DATA) as f: -# cls.data = json.load(f) - -# def test_to_version_ranges(self): -# data = [ -# { -# "version_affected": "?=", -# "version_value": "1.3.0", -# }, -# { -# "version_affected": "=", -# "version_value": "1.3.1", -# }, -# { -# "version_affected": "<", -# "version_value": "1.3.2", -# }, -# ] -# fixed_version_ranges, affected_version_ranges = self.data_src.to_version_ranges(data) - -# # Check fixed packages -# assert [ -# VersionRange.from_scheme_version_spec_string("semver", ">=1.3.2") -# ] == fixed_version_ranges - -# # Check vulnerable packages -# assert [ -# VersionRange.from_scheme_version_spec_string("semver", "==1.3.0"), -# VersionRange.from_scheme_version_spec_string("semver", "==1.3.1"), -# ] == affected_version_ranges - -# def test_to_advisory(self): -# expected_advisories = [ -# AdvisoryData( -# summary="A serious problem exists when a client sends a large number of " -# "headers with the same header name. Apache uses up memory faster than the " -# "amount of memory required to simply store the received data itself. That " -# "is, memory use increases faster and faster as more headers are received, " -# "rather than increasing at a constant rate. This makes a denial of service " -# "attack based on this method more effective than methods which cause Apache" -# " to use memory at a constant rate, since the attacker has to send less data.", -# affected_packages=[ -# AffectedPackage( -# vulnerable_package=PackageURL( -# type="apache", -# name="httpd", -# version="1.3.0", -# ), -# ), -# AffectedPackage( -# vulnerable_package=PackageURL( -# type="apache", -# name="httpd", -# version="1.3.1", -# ), -# ), -# ], -# references=[ -# Reference( -# url="https://httpd.apache.org/security/json/CVE-1999-1199.json", -# severities=[ -# VulnerabilitySeverity( -# system=severity_systems.APACHE_HTTPD, -# value="important", -# ), -# ], -# reference_id="CVE-1999-1199", -# ), -# ], -# vulnerability_id="CVE-1999-1199", -# ) -# ] -# found_advisories = [self.data_src.to_advisory(self.data)] -# found_advisories = list(map(AdvisoryData.normalized, found_advisories)) -# expected_advisories = list(map(AdvisoryData.normalized, expected_advisories)) -# assert sorted(found_advisories) == sorted(expected_advisories) - -# def test_misc_01(self): -# print("\nHello!\n") -# assert True == True - - -# def test_misc_01(): -# print("\nHello!\n") -# assert True == True - - -# def test_to_advisory(): -# with open(os.path.join(TEST_DATA, "CVE-1999-1199.json")) as f: -# raw_data = json.load(f) - -# print("\n\nraw_data = \n{}\n".format(raw_data)) - -# # print("\npretty raw_data = {}".format(json.dumps(raw_data, indent=4))) - -# # The following throws an error: TypeError: to_advisory() missing 1 required positional argument: 'data' -# # presumably because it also needs to pass 'self'? -# advisories = ApacheHTTPDImporter.to_advisory(raw_data) -# # result = [data.to_dict() for data in advisories] -# # expected_file = os.path.join(TEST_DATA, f"parse-advisory-postgresql-expected.json") -# # util_tests.check_results_against_json(result, expected_file) - - -class TestApacheHTTPDImporter(TestCase): - base_url = "https://httpd.apache.org/security/json/" - - def test_to_advisory_in_class(self): - # with open(os.path.join(TEST_DATA, "CVE-1999-1199.json")) as f: - with open(os.path.join(TEST_DATA, "CVE-2021-44224.json")) as f: - raw_data = json.load(f) - - # print("\n\nraw_data = \n{}\n".format(raw_data)) - print( - "\n\nJSON input file CVE-1999-1199.json = \n\n{}".format(json.dumps(raw_data, indent=2)) +def test_to_version_ranges(): + data = [ + { + "version_affected": "?=", + "version_value": "1.3.0", + }, + { + "version_affected": "=", + "version_value": "1.3.1", + }, + { + "version_affected": "<=", + "version_value": "2.3.4", + }, + ] + affected_version_range = ApacheHTTPDImporter().to_version_ranges(data) + + # Check vulnerable packages + assert ( + GenericVersionRange( + constraints=( + VersionConstraint(comparator="=", version=SemverVersion(string="1.3.0")), + VersionConstraint(comparator="=", version=SemverVersion(string="1.3.1")), + VersionConstraint(comparator="<=", version=SemverVersion(string="2.3.4")), + ) ) + == affected_version_range + ) - advisory = ApacheHTTPDImporter.to_advisory(self, raw_data) - - print("\n\nJSON input file to_advisory() = \n\n{}\n".format(advisory)) - - # print("advisory.aliases = {}\n".format(advisory.aliases)) - - # print("advisory.summary = {}\n".format(advisory.summary)) - - # print("advisory.affected_packages = {}\n".format(advisory.affected_packages)) - # print("advisory.references = {}\n".format(advisory.references)) - # for ref in advisory.references: - # print("\treference = {}\n".format(ref)) +def test_to_advisory_CVE_1999_1199(): + with open(os.path.join(TEST_DATA, "CVE-1999-1199.json")) as f: + data = json.load(f) - # print("advisory.date_published = {}\n".format(advisory.date_published)) + advisories = ApacheHTTPDImporter().to_advisory(data) + result = advisories.to_dict() + expected_file = os.path.join(TEST_DATA, f"to-advisory-CVE-1999-1199-apache-httpd-expected.json") + util_tests.check_results_against_json(result, expected_file) - result = advisory.to_dict() - # print("result = {}\n".format(result)) +def test_to_advisory_CVE_2021_44224(): + with open(os.path.join(TEST_DATA, "CVE-2021-44224.json")) as f: + data = json.load(f) - print("\nadvisory.to_dict() = \n\n{}\n".format(json.dumps(result, indent=2))) + advisories = ApacheHTTPDImporter().to_advisory(data) + result = advisories.to_dict() + expected_file = os.path.join( + TEST_DATA, f"to-advisory-CVE-2021-44224-apache-httpd-expected.json" + ) + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-1999-1199-apache-httpd-expected.json b/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-1999-1199-apache-httpd-expected.json new file mode 100644 index 000000000..815b7a460 --- /dev/null +++ b/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-1999-1199-apache-httpd-expected.json @@ -0,0 +1,34 @@ +{ + "aliases": [ + "CVE-1999-1199" + ], + "summary": "A serious problem exists when a client sends a large number of headers with the same header name. Apache uses up memory faster than the amount of memory required to simply store the received data itself. That is, memory use increases faster and faster as more headers are received, rather than increasing at a constant rate. This makes a denial of service attack based on this method more effective than methods which cause Apache to use memory at a constant rate, since the attacker has to send less data.", + "affected_packages": [ + { + "package": { + "type": "generic", + "namespace": null, + "name": "apache_httpd", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:generic/1.3.0|1.3.1", + "fixed_version": null + } + ], + "references": [ + { + "reference_id": "CVE-1999-1199", + "url": "https://httpd.apache.org/security/json/CVE-1999-1199.json", + "severities": [ + { + "system": "apache_httpd", + "value": "important", + "scoring_elements": "" + } + ] + } + ], + "date_published": null +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-2021-44224-apache-httpd-expected.json b/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-2021-44224-apache-httpd-expected.json new file mode 100644 index 000000000..5387b0716 --- /dev/null +++ b/vulnerabilities/tests/test_data/apache_httpd/to-advisory-CVE-2021-44224-apache-httpd-expected.json @@ -0,0 +1,34 @@ +{ + "aliases": [ + "CVE-2021-44224" + ], + "summary": "A crafted URI sent to httpd configured as a forward proxy (ProxyRequests on) can cause a crash (NULL pointer dereference) or, for configurations mixing forward and reverse proxy declarations, can allow for requests to be directed to a declared Unix Domain Socket endpoint (Server Side Request Forgery).\n\nThis issue affects Apache HTTP Server 2.4.7 up to 2.4.51 (included).", + "affected_packages": [ + { + "package": { + "type": "generic", + "namespace": null, + "name": "apache_httpd", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:generic/>=2.4.7|<=2.4.51", + "fixed_version": null + } + ], + "references": [ + { + "reference_id": "CVE-2021-44224", + "url": "https://httpd.apache.org/security/json/CVE-2021-44224.json", + "severities": [ + { + "system": "apache_httpd", + "value": "moderate", + "scoring_elements": "" + } + ] + } + ], + "date_published": null +} \ No newline at end of file