From b5067a3e09db10b9f73bb786bc06ccac1f5f5317 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sun, 9 May 2021 16:55:37 +0530 Subject: [PATCH] Dump yaml in favor of saneyaml saneyaml, reorder and refactor imports Signed-off-by: Hritik Vijay --- requirements.txt | 3 ++- vulnerabilities/helpers.py | 6 +++--- vulnerabilities/importers/alpine_linux.py | 4 ++-- vulnerabilities/importers/elixir_security.py | 10 ++++------ vulnerabilities/importers/istio.py | 16 +++++++++------- vulnerabilities/importers/suse_backports.py | 7 +++---- vulnerabilities/importers/suse_scores.py | 6 +++--- vulnerabilities/tests/test_alpine.py | 10 ++++------ vulnerabilities/tests/test_istio.py | 3 +-- 9 files changed, 31 insertions(+), 34 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d96782c6..32aecc5be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,4 +57,5 @@ zipp==0.6.0 requests==2.23.0 toml==0.10.2 PyYAML==5.4 -freezegun==1.1.0 \ No newline at end of file +freezegun==1.1.0 +saneyaml==0.5.2 diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index c39b32ee0..37fa7d1cb 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -28,9 +28,9 @@ from typing import List import requests +import saneyaml import toml import urllib3 -import yaml from packageurl import PackageURL from univers.versions import version_class_by_package_type @@ -49,7 +49,7 @@ class AffectedPackage: def load_yaml(path): with open(path) as f: - return yaml.safe_load(f) + return saneyaml.load(f) def load_json(path): @@ -64,7 +64,7 @@ def load_toml(path): def fetch_yaml(url): response = requests.get(url) - return yaml.safe_load(response.content) + return saneyaml.load(response.content) # FIXME: this is NOT how etags work . diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 758365a1f..3159689de 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -28,7 +28,7 @@ from typing import Set import requests -import yaml +import saneyaml from bs4 import BeautifulSoup from packageurl import PackageURL from schema import Or @@ -113,7 +113,7 @@ def updated_advisories(self) -> Set[Advisory]: def _process_link(self, link) -> List[Advisory]: advisories = [] yaml_response = requests.get(link).content - record = yaml.safe_load(yaml_response) + record = saneyaml.load(yaml_response) if record["packages"] is None: return advisories diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index b491ed529..fcb3e6c42 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -19,21 +19,19 @@ # for any legal advice. # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - import asyncio -from typing import List, Set +from typing import Set -import yaml +from packageurl import PackageURL from univers.version_specifier import VersionSpecifier from univers.versions import SemverVersion -from packageurl import PackageURL -from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Reference -from vulnerabilities.package_managers import HexVersionAPI from vulnerabilities.helpers import load_yaml from vulnerabilities.helpers import nearest_patched_package +from vulnerabilities.package_managers import HexVersionAPI class ElixirSecurityDataSource(GitDataSource): diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9b88054a6..509bde08d 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -19,19 +19,21 @@ # for any legal advice. # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - import asyncio import re -from typing import List, Set -import yaml +from typing import List +from typing import Set +import saneyaml +from packageurl import PackageURL from univers.version_specifier import VersionSpecifier from univers.versions import SemverVersion -from packageurl import PackageURL -from vulnerabilities.data_source import Advisory, GitDataSource, Reference -from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import Reference from vulnerabilities.helpers import nearest_patched_package +from vulnerabilities.package_managers import GitHubTagsAPI is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match @@ -89,7 +91,7 @@ def get_data_from_yaml_lines(self, yaml_lines): 'cves': '[CVE-2019-12243]'} """ - return yaml.safe_load("\n".join(yaml_lines)) + return saneyaml.load("\n".join(yaml_lines)) def get_yaml_lines(self, lines): """The istio advisory file contains lines similar to yaml format . diff --git a/vulnerabilities/importers/suse_backports.py b/vulnerabilities/importers/suse_backports.py index f03d88676..c80d25969 100644 --- a/vulnerabilities/importers/suse_backports.py +++ b/vulnerabilities/importers/suse_backports.py @@ -19,13 +19,12 @@ # for any legal advice. # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import yaml import dataclasses import requests -from packageurl import PackageURL +import saneyaml from bs4 import BeautifulSoup +from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource @@ -65,7 +64,7 @@ def _fetch_yaml(self, url): try: resp = requests.get(url) resp.raise_for_status() - return yaml.safe_load(resp.content) + return saneyaml.load(resp.content) except requests.HTTPError: return {} diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index cd01abe90..3d853b0ce 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -45,7 +45,7 @@ def to_advisory(score_data): for cvss_score in score_data[cve_id]["cvss"]: score = None vector = None - if cvss_score["version"] == 2.0: + if cvss_score["version"] == "2.0": score = VulnerabilitySeverity( system=scoring_systems["cvssv2"], value=str(cvss_score["score"]) ) @@ -53,7 +53,7 @@ def to_advisory(score_data): system=scoring_systems["cvssv2_vector"], value=str(cvss_score["vector"]) ) - elif cvss_score["version"] == 3: + elif cvss_score["version"] == "3": score = VulnerabilitySeverity( system=scoring_systems["cvssv3"], value=str(cvss_score["score"]) ) @@ -61,7 +61,7 @@ def to_advisory(score_data): system=scoring_systems["cvssv3_vector"], value=str(cvss_score["vector"]) ) - elif cvss_score["version"] == 3.1: + elif cvss_score["version"] == "3.1": score = VulnerabilitySeverity( system=scoring_systems["cvssv3.1"], value=str(cvss_score["score"]) ) diff --git a/vulnerabilities/tests/test_alpine.py b/vulnerabilities/tests/test_alpine.py index 864fb7ebe..8b1e75a0a 100644 --- a/vulnerabilities/tests/test_alpine.py +++ b/vulnerabilities/tests/test_alpine.py @@ -20,15 +20,13 @@ # for any legal advice. # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - import os -import yaml from unittest import TestCase -from unittest.mock import patch, MagicMock - -from packageurl import PackageURL +from unittest.mock import MagicMock +from unittest.mock import patch -from vulnerabilities.data_source import Advisory, Reference +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference from vulnerabilities.importers.alpine_linux import AlpineDataSource diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index 875431756..f17dc64ed 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -20,7 +20,6 @@ # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import datetime import os from collections import OrderedDict from unittest import TestCase @@ -70,7 +69,7 @@ def test_get_data_from_md(self): "cvss": "8.9", "vector": "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C", "releases": ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"], - "publishdate": datetime.date(2019, 5, 28), + "publishdate": "2019-05-28", } assert expected_data == actual_data