From 303efca5669dd7eaf32b75dbd69dce8995fc9a63 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 20 Jun 2022 21:29:42 +0530 Subject: [PATCH 01/36] add initial config for vulntotal Signed-off-by: Keshav Priyadarshi --- vulntotal/validator.py | 53 ++++++++++++++++++++++++++ vulntotal/validator_runner.py | 22 +++++++++++ vulntotal/validators/__init__.py | 27 +++++++++++++ vulntotal/validators/vulnerablecode.py | 22 +++++++++++ 4 files changed, 124 insertions(+) create mode 100644 vulntotal/validator.py create mode 100644 vulntotal/validator_runner.py create mode 100644 vulntotal/validators/__init__.py create mode 100644 vulntotal/validators/vulnerablecode.py diff --git a/vulntotal/validator.py b/vulntotal/validator.py new file mode 100644 index 000000000..07b3364b5 --- /dev/null +++ b/vulntotal/validator.py @@ -0,0 +1,53 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import dataclasses +import json +from typing import Iterable +from typing import List + +from vulnerabilities.utils import classproperty + + +@dataclasses.dataclass(order=True) +class VendorData: + raw_dump: str = "" + aliases: List[str] = dataclasses.field(default_factory=list) + affected_versions: List[str] = dataclasses.field(default_factory=list) + fixed_versions: List[str] = dataclasses.field(default_factory=list) + + def to_dict(self): + return { + "affected_versions": self.affected_versions, + "fixed_versions": self.fixed_versions, + "aliases": self.aliases, + "raw_dump": self.raw_dump, + } + + +class Validator: + def validator_advisory(self, purl) -> Iterable[VendorData]: + """ + Yield VendorData object corresponding to vendor + """ + return NotImplementedError diff --git a/vulntotal/validator_runner.py b/vulntotal/validator_runner.py new file mode 100644 index 000000000..4a3bf33bb --- /dev/null +++ b/vulntotal/validator_runner.py @@ -0,0 +1,22 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. diff --git a/vulntotal/validators/__init__.py b/vulntotal/validators/__init__.py new file mode 100644 index 000000000..f599c5994 --- /dev/null +++ b/vulntotal/validators/__init__.py @@ -0,0 +1,27 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + + +VALIDATORS_REGISTRY = [] + +VALIDATORS_REGISTRY = {x.__module__.split(".")[-1]: x for x in VALIDATORS_REGISTRY} diff --git a/vulntotal/validators/vulnerablecode.py b/vulntotal/validators/vulnerablecode.py new file mode 100644 index 000000000..4a3bf33bb --- /dev/null +++ b/vulntotal/validators/vulnerablecode.py @@ -0,0 +1,22 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. From 2176cb119614b0381ebd56551779266747f9a871 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 19 Jul 2022 18:38:03 +0530 Subject: [PATCH 02/36] don't include `raw_dump` in `VendorData` Signed-off-by: Keshav Priyadarshi --- vulntotal/validator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vulntotal/validator.py b/vulntotal/validator.py index 07b3364b5..3118dddb1 100644 --- a/vulntotal/validator.py +++ b/vulntotal/validator.py @@ -31,7 +31,6 @@ @dataclasses.dataclass(order=True) class VendorData: - raw_dump: str = "" aliases: List[str] = dataclasses.field(default_factory=list) affected_versions: List[str] = dataclasses.field(default_factory=list) fixed_versions: List[str] = dataclasses.field(default_factory=list) @@ -41,13 +40,18 @@ def to_dict(self): "affected_versions": self.affected_versions, "fixed_versions": self.fixed_versions, "aliases": self.aliases, - "raw_dump": self.raw_dump, } class Validator: + _raw_dump = [] + def validator_advisory(self, purl) -> Iterable[VendorData]: """ Yield VendorData object corresponding to vendor """ return NotImplementedError + + @property + def raw_dump(self): + return self._raw_dump From 922859f3c198eb0e78b51f0f4600bbb872059bed Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 25 Jul 2022 05:45:32 +0530 Subject: [PATCH 03/36] add supported ecosystem Signed-off-by: Keshav Priyadarshi --- vulntotal/validator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vulntotal/validator.py b/vulntotal/validator.py index 3118dddb1..b391f98da 100644 --- a/vulntotal/validator.py +++ b/vulntotal/validator.py @@ -52,6 +52,16 @@ def validator_advisory(self, purl) -> Iterable[VendorData]: """ return NotImplementedError + @classmethod + def support_ecosystem(cls): + """ + Return dictionary containing supported ecosystem + { + "PURL equivalent ecosystem" : "Validator ecosystem", + } + """ + return NotImplementedError + @property def raw_dump(self): return self._raw_dump From 78dd5ae7f736874b05764b935968e2e79559feb1 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 4 Aug 2022 23:31:07 +0530 Subject: [PATCH 04/36] drop Validator in favour of DataSource Signed-off-by: Keshav Priyadarshi --- vulntotal/{validators => datasources}/__init__.py | 4 ++-- .../{validators => datasources}/vulnerablecode.py | 0 vulntotal/validator.py | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) rename vulntotal/{validators => datasources}/__init__.py (92%) rename vulntotal/{validators => datasources}/vulnerablecode.py (100%) diff --git a/vulntotal/validators/__init__.py b/vulntotal/datasources/__init__.py similarity index 92% rename from vulntotal/validators/__init__.py rename to vulntotal/datasources/__init__.py index f599c5994..a2b13a650 100644 --- a/vulntotal/validators/__init__.py +++ b/vulntotal/datasources/__init__.py @@ -22,6 +22,6 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. -VALIDATORS_REGISTRY = [] +DATASOURCE_REGISTRY = [] -VALIDATORS_REGISTRY = {x.__module__.split(".")[-1]: x for x in VALIDATORS_REGISTRY} +DATASOURCE_REGISTRY = {x.__module__.split(".")[-1]: x for x in DATASOURCE_REGISTRY} diff --git a/vulntotal/validators/vulnerablecode.py b/vulntotal/datasources/vulnerablecode.py similarity index 100% rename from vulntotal/validators/vulnerablecode.py rename to vulntotal/datasources/vulnerablecode.py diff --git a/vulntotal/validator.py b/vulntotal/validator.py index b391f98da..ec2f9e395 100644 --- a/vulntotal/validator.py +++ b/vulntotal/validator.py @@ -43,21 +43,22 @@ def to_dict(self): } -class Validator: - _raw_dump = [] +class DataSource: + def __init__(self): + self._raw_dump = [] - def validator_advisory(self, purl) -> Iterable[VendorData]: + def datasource_advisory(self, purl) -> Iterable[VendorData]: """ - Yield VendorData object corresponding to vendor + Yield VendorData object corresponding to DataSource """ return NotImplementedError @classmethod - def support_ecosystem(cls): + def supported_ecosystem(cls): """ Return dictionary containing supported ecosystem { - "PURL equivalent ecosystem" : "Validator ecosystem", + "PURL equivalent ecosystem" : "DataSource ecosystem", } """ return NotImplementedError From fe20e49c9664a7a1052bf6fdc3570708aec0be00 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 30 Aug 2022 00:10:48 +0530 Subject: [PATCH 05/36] adopt new CLI Signed-off-by: Keshav Priyadarshi --- vulntotal/vulntotal_cli.py | 382 +++++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100755 vulntotal/vulntotal_cli.py diff --git a/vulntotal/vulntotal_cli.py b/vulntotal/vulntotal_cli.py new file mode 100755 index 000000000..667b706df --- /dev/null +++ b/vulntotal/vulntotal_cli.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import concurrent.futures +import json +import pydoc +import sys + +import click +import yaml +from packageurl import PackageURL +from texttable import Texttable + +from vulntotal.datasources import DATASOURCE_REGISTRY +from vulntotal.validator import VendorData + + +@click.command() +@click.option( + "-l", + "--list", + "list_source", + is_flag=True, + multiple=False, + required=False, + help="Lists all the available DataSources.", +) +@click.option( + "-e", + "--enable", + "enable", + hidden=True, + multiple=True, + type=click.Choice(DATASOURCE_REGISTRY.keys()), + required=False, + help="Enable these datasource/s only.", +) +@click.option( + "-d", + "--disable", + "disable", + hidden=True, + multiple=True, + type=click.Choice(DATASOURCE_REGISTRY.keys()), + required=False, + help="Disable these datasource/s.", +) +@click.option( + "--ecosystem", + "ecosystem", + hidden=True, + is_flag=True, + required=False, + help="Lists ecosystem supported by active DataSources", +) +@click.option( + "--raw", + "raw_output", + is_flag=True, + hidden=True, + multiple=False, + required=False, + help="List of all the raw response from DataSources.", +) +@click.option( + "--no-threading", + "no_threading", + is_flag=True, + hidden=True, + multiple=False, + required=False, + help="Run DataSources sequentially.", +) +@click.option( + "-p", + "--pagination", + "pagination", + is_flag=True, + hidden=True, + multiple=False, + required=False, + help="Enable default pagination.", +) +@click.option( + "--json", + "json_output", + type=click.File("w"), + required=False, + metavar="FILE", + help="Write output as pretty-printed JSON to FILE. ", +) +@click.option( + "--yaml", + "yaml_output", + type=click.File("w"), + required=False, + metavar="FILE", + help="Write output as YAML to FILE. ", +) +@click.option( + "--no-group", + "no_group", + is_flag=True, + hidden=True, + multiple=False, + required=False, + help="Don't group by CVE.", +) +@click.argument("purl", required=False) +@click.help_option("-h", "--help") +def handler( + purl, + list_source, + enable, + disable, + ecosystem, + raw_output, + no_threading, + pagination, + json_output, + yaml_output, + no_group, +): + """ + Runs the PURL through all the available datasources and group vulnerability by CVEs. + Use the special '-' file name to print JSON or YAML results on screen/stdout. + """ + active_datasource = ( + get_enabled_datasource(enable) + if enable + else (get_undisabled_datasource(disable) if disable else DATASOURCE_REGISTRY) + ) + + if list_source: + list_datasources() + + elif not active_datasource: + click.echo("No datasources available!", err=True) + + elif ecosystem: + list_supported_ecosystem(active_datasource) + + elif raw_output: + if purl: + get_raw_response(purl, active_datasource) + + elif json_output: + write_json_output(purl, active_datasource, json_output, no_threading) + + elif yaml_output: + write_yaml_output(purl, active_datasource, yaml_output, no_threading) + + elif no_group: + prettyprint(purl, active_datasource, pagination, no_threading) + + elif purl: + prettyprint_group_by_cve(purl, active_datasource, pagination, no_threading) + + +def get_valid_datasources(datasources): + valid_datasources = {} + unknown_datasources = [] + for datasource in datasources: + key = datasource.lower() + try: + valid_datasources[key] = DATASOURCE_REGISTRY[key] + except KeyError: + unknown_datasources.append(key) + if unknown_datasources: + raise CommandError(f"Unknown datasource: {unknown_datasources}") + return valid_datasources + + +def get_undisabled_datasource(datasources): + disabled = get_valid_datasources(datasources) + return {key: value for key, value in DATASOURCE_REGISTRY.items() if key not in disabled} + + +def get_enabled_datasource(datasources): + return get_valid_datasources(datasources) + + +def list_datasources(): + datasources = [x.upper() for x in list(DATASOURCE_REGISTRY)] + click.echo("Currently supported datasources:") + click.echo("\n".join(sorted(datasources))) + + +def list_supported_ecosystem(datasources): + ecosystems = [] + for key, datasource in datasources.items(): + vendor_supported_ecosystem = datasource.supported_ecosystem() + ecosystems.extend([x.upper() for x in vendor_supported_ecosystem.keys()]) + + active_datasource = [x.upper() for x in datasources.keys()] + click.echo("Active DataSources: %s\n" % ", ".join(sorted(active_datasource))) + click.echo("Ecosystem supported by active datasources") + click.echo("\n".join(sorted(set(ecosystems)))) + + +def formatted_row(datasource, advisory): + aliases = "\n".join(advisory.aliases) + affected = " ".join(advisory.affected_versions) + fixed = " ".join(advisory.fixed_versions) + return [datasource.upper(), aliases, affected, fixed] + + +def get_raw_response(purl, datasources): + all_raw_responses = {} + for key, datasource in datasources.items(): + vendor = datasource() + vendor_advisories = list(vendor.datasource_advisory(PackageURL.from_string(purl))) + all_raw_responses[key] = vendor.raw_dump + click.echo(json.dumps(all_raw_responses, indent=2)) + + +def run_datasources(purl, datasources, no_threading=False): + vulnerabilities = {} + if not no_threading: + with concurrent.futures.ThreadPoolExecutor(max_workers=len(datasources)) as executor: + future_to_advisory = { + executor.submit( + datasource().datasource_advisory, PackageURL.from_string(purl) + ): datasource + for key, datasource in datasources.items() + } + for future in concurrent.futures.as_completed(future_to_advisory): + vendor = future_to_advisory[future].__name__[:-10].lower() + try: + vendor_advisories = future.result() + vulnerabilities[vendor] = [] + if vendor_advisories: + vulnerabilities[vendor].extend([advisory for advisory in vendor_advisories]) + except Exception as exc: + click.echo("%s generated an exception: %s" % (vendor, exc)) + else: + for key, datasource in datasources.items(): + vendor_advisories = datasource().datasource_advisory(PackageURL.from_string(purl)) + vulnerabilities[key] = [] + if vendor_advisories: + vulnerabilities[key].extend([advisory for advisory in vendor_advisories]) + + return vulnerabilities + + +class VendorDataEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, VendorData): + return obj.to_dict() + return json.JSONEncoder.default(self, obj) + + +def write_json_output(purl, datasources, json_output, no_threading): + vulnerabilities = run_datasources(purl, datasources, no_threading) + return json.dump(vulnerabilities, json_output, cls=VendorDataEncoder, indent=2) + + +def noop(self, *args, **kw): + pass + + +yaml.emitter.Emitter.process_tag = noop + + +def write_yaml_output(purl, datasources, yaml_output, no_threading): + vulnerabilities = run_datasources(purl, datasources, no_threading) + return yaml.dump(vulnerabilities, yaml_output, default_flow_style=False, indent=2) + + +def prettyprint(purl, datasources, pagination, no_threading): + vulnerabilities = run_datasources(purl, datasources, no_threading) + if not vulnerabilities: + return + + active_datasource = ", ".join(sorted([x.upper() for x in datasources.keys()])) + metadata = f"PURL: {purl}\nActive DataSources: {active_datasource}\n\n" + + table = Texttable() + table.set_cols_dtype(["t", "t", "t", "t"]) + table.set_cols_align(["c", "l", "l", "l"]) + table.set_cols_valign(["t", "t", "a", "t"]) + table.header(["DATASOURCE", "ALIASES", "AFFECTED", "FIXED"]) + + for datasource, advisories in vulnerabilities.items(): + if not advisories: + table.add_row([datasource.upper(), "", "", ""]) + continue + + for advisory in advisories: + table.add_row(formatted_row(datasource, advisory)) + + pydoc.pager(metadata + table.draw()) if pagination else click.echo(metadata + table.draw()) + + +def group_by_cve(vulnerabilities): + grouped_by_cve = {} + nocve = [] + noadvisory = [] + for datasource, advisories in vulnerabilities.items(): + if not advisories: + noadvisory.append([datasource.upper(), "", "", ""]) + + for advisory in advisories: + cve = next((x for x in advisory.aliases if x.startswith("CVE")), None) + if not cve: + nocve.append(formatted_row(datasource, advisory)) + continue + if cve not in grouped_by_cve: + grouped_by_cve[cve] = [] + grouped_by_cve[cve].append(formatted_row(datasource, advisory)) + grouped_by_cve["NOCVE"] = nocve + grouped_by_cve["NOADVISORY"] = noadvisory + return grouped_by_cve + + +def prettyprint_group_by_cve(purl, datasources, pagination, no_threading): + vulnerabilities = run_datasources(purl, datasources, no_threading) + if not vulnerabilities: + return + grouped_by_cve = group_by_cve(vulnerabilities) + + active_datasource = ", ".join(sorted([x.upper() for x in datasources.keys()])) + metadata = f"PURL: {purl}\nActive DataSources: {active_datasource}\n\n" + + table = Texttable() + table.set_cols_dtype(["a", "a", "a", "a", "a"]) + table.set_cols_align(["l", "l", "l", "l", "l"]) + table.set_cols_valign(["t", "t", "t", "a", "t"]) + table.header(["CVE", "DATASOURCE", "ALIASES", "AFFECTED", "FIXED"]) + + for cve, advisories in grouped_by_cve.items(): + for count, advisory in enumerate(advisories): + table.add_row([cve] + advisory) + + pydoc.pager(metadata + table.draw()) if pagination else click.echo(metadata + table.draw()) + + +if __name__ == "__main__": + handler() + +""" +Advanced Usage: vulntotal_cli.py [OPTIONS] [PURL] + + Runs the PURL through all the available datasources and group vulnerability + by CVEs. Use the special '-' file name to print JSON or YAML results on + screen/stdout. + +Options: + -l, --list Lists all the available DataSources. + --json FILE Write output as pretty-printed JSON to FILE. + --yaml FILE Write output as YAML to FILE. + -e, --enable Enable these datasource/s only. + -d, --disable Disable these datasource/s. + --ecosystem Lists ecosystem supported by active DataSources + --raw List of all the raw response from DataSources. + --no-threading Run DataSources sequentially. + -p, --pagination Enable default pagination. + --no-group Don't group by CVE. + -h, --help Show this message and exit. +""" From e30131fa14d66e139796923f0a5bcffd72e18096 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:11:06 +0530 Subject: [PATCH 06/36] add Deps Datasource Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/deps.py | 122 ++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 vulntotal/datasources/deps.py diff --git a/vulntotal/datasources/deps.py b/vulntotal/datasources/deps.py new file mode 100644 index 000000000..e41cb1711 --- /dev/null +++ b/vulntotal/datasources/deps.py @@ -0,0 +1,122 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +import logging +from typing import Iterable +from urllib.parse import quote + +import requests +from packageurl import PackageURL + +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData + +logger = logging.getLogger(__name__) + + +class DepsDataSource(DataSource): + spdx_license_expression = "TODO" + license_url = "TODO" + + def fetch_json_response(self, url): + response = requests.get(url) + if not response.status_code == 200 or response.text == "Not Found": + logger.error(f"Error while fetching {url}") + return + return response.json() + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + payload = generate_meta_payload(purl) + response = self.fetch_json_response(payload) + if response: + advisories = parse_advisories_from_meta(response) + if advisories: + for advisory in advisories: + advisory_payload = generate_advisory_payload(advisory) + fetched_advisory = self.fetch_json_response(advisory_payload) + self._raw_dump.append(fetched_advisory) + if fetched_advisory: + return parse_advisory(fetched_advisory) + + @classmethod + def supported_ecosystem(cls): + return { + "npm": "npm", + "maven": "maven", + "golang": "go", + "pypi": "pypi", + "cargo": "cargo", + # Coming soon + # "nuget": "nuget", + } + + +def parse_advisory(advisory) -> Iterable[VendorData]: + affected_versions = [event["version"] for event in advisory["packages"][0]["versionsAffected"]] + fixed_versions = [event["version"] for event in advisory["packages"][0]["versionsUnaffected"]] + yield VendorData( + aliases=sorted(list(set(advisory["aliases"]))), + affected_versions=sorted(list(set(affected_versions))), + fixed_versions=sorted(list(set(fixed_versions))), + ) + + +def parse_advisories_from_meta(advisories_metadata): + advisories = [] + if "dependencies" in advisories_metadata and advisories_metadata["dependencies"]: + for dependency in advisories_metadata["dependencies"]: + if dependency["advisories"]: + advisories.extend(dependency["advisories"]) + return advisories + + +def generate_advisory_payload(advisory_meta): + url_advisory = "https://deps.dev/_/advisory/{source}/{sourceID}" + return url_advisory.format(source=advisory_meta["source"], sourceID=advisory_meta["sourceID"]) + + +def generate_meta_payload(purl): + url_advisories_meta = "https://deps.dev/_/s/{ecosystem}/p/{package}/v/{version}/dependencies" + supported_ecosystem = DepsDataSource.supported_ecosystem() + if purl.type in supported_ecosystem: + purl_version = purl.version + purl_name = purl.name + + if purl.type == "maven": + if not purl.namespace: + logger.error(f"Invalid Maven PURL {str(purl)}") + return + purl_name = quote(f"{purl.namespace}:{purl.name}", safe="") + + elif purl.type == "golang": + if purl.namespace: + purl_name = quote(f"{purl.namespace}/{purl.name}", safe="") + if not purl_version.startswith("v"): + purl_version = f"v{purl_version}" + + return url_advisories_meta.format( + ecosystem=supported_ecosystem[purl.type], + package=purl_name, + version=purl_version, + ) From 3235316c242e1c16e856c94fea5a6b6a01872a5b Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:12:50 +0530 Subject: [PATCH 07/36] add tests for Deps Datasource Signed-off-by: Keshav Priyadarshi --- .../test_data/deps/advisories_metadata.json | 179 +++++++++++++ .../test_data/deps/advisories_metadata.txt | 208 +++++++++++++++ vulntotal/tests/test_data/deps/advisory.json | 247 ++++++++++++++++++ .../deps/parse_advisory-expected.json | 60 +++++ .../parsed_advisories_metadata-expected.json | 179 +++++++++++++ .../deps/payloads_advisories-expected.json | 8 + .../deps/payloads_meta-expected.json | 7 + vulntotal/tests/test_data/deps/purls.txt | 5 + vulntotal/tests/test_deps.py | 69 +++++ 9 files changed, 962 insertions(+) create mode 100644 vulntotal/tests/test_data/deps/advisories_metadata.json create mode 100644 vulntotal/tests/test_data/deps/advisories_metadata.txt create mode 100644 vulntotal/tests/test_data/deps/advisory.json create mode 100644 vulntotal/tests/test_data/deps/parse_advisory-expected.json create mode 100644 vulntotal/tests/test_data/deps/parsed_advisories_metadata-expected.json create mode 100644 vulntotal/tests/test_data/deps/payloads_advisories-expected.json create mode 100644 vulntotal/tests/test_data/deps/payloads_meta-expected.json create mode 100644 vulntotal/tests/test_data/deps/purls.txt create mode 100644 vulntotal/tests/test_deps.py diff --git a/vulntotal/tests/test_data/deps/advisories_metadata.json b/vulntotal/tests/test_data/deps/advisories_metadata.json new file mode 100644 index 000000000..8f0783ed0 --- /dev/null +++ b/vulntotal/tests/test_data/deps/advisories_metadata.json @@ -0,0 +1,179 @@ +[ + { + "source": "GHSA", + "sourceID": "GHSA-g3rq-g295-4j3m", + "sourceURL": "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "title": "Regular Expression Denial of Service (ReDoS) in Jinja2", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDOS vulnerability of the regex is mainly due to the sub-pattern [a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+ This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://nvd.nist.gov/vuln/detail/CVE-2020-28493", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://security.gentoo.org/glsa/202107-19", + "https://github.com/advisories/GHSA-g3rq-g295-4j3m" + ], + "severity": "MEDIUM", + "gitHubSeverity": "MODERATE", + "scoreV3": 5.3, + "aliases": [ + "CVE-2020-28493" + ], + "disclosedAt": 1616189285, + "observedAt": 1650328213 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-8", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-8", + "title": "PYSEC-2014-8", + "description": "The default configuration for bccache.FileSystemBytecodeCache in Jinja2 before 2.7.2 does not properly create temporary files, which allows local users to gain privileges via a crafted .cache file with a name starting with __jinja2_ in /tmp.", + "referenceURLs": [ + "http://advisories.mageia.org/MGASA-2014-0028.html", + "http://jinja.pocoo.org/docs/changelog/", + "http://openwall.com/lists/oss-security/2014/01/10/2", + "http://openwall.com/lists/oss-security/2014/01/10/3", + "http://rhn.redhat.com/errata/RHSA-2014-0747.html", + "http://rhn.redhat.com/errata/RHSA-2014-0748.html", + "http://secunia.com/advisories/56287", + "http://secunia.com/advisories/58783", + "http://secunia.com/advisories/58918", + "http://secunia.com/advisories/59017", + "http://secunia.com/advisories/60738", + "http://secunia.com/advisories/60770", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "http://www.mandriva.com/security/advisories?name=MDVSA-2014:096", + "https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-8.yaml", + "https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-1402" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-82", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-82", + "title": "PYSEC-2014-82", + "description": "FileSystemBytecodeCache in Jinja2 2.7.2 does not properly create temporary directories, which allows local users to gain privileges by pre-creating a temporary directory with a user's uid. NOTE: this vulnerability exists because of an incomplete fix for CVE-2014-1402.", + "referenceURLs": [ + "http://seclists.org/oss-sec/2014/q1/73", + "http://secunia.com/advisories/56328", + "http://secunia.com/advisories/60738", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/mitsuhiko/jinja2", + "https://github.com/mitsuhiko/jinja2/commit/acb672b6a179567632e032f547582f30fa2f4aa7", + "https://github.com/mitsuhiko/jinja2/pull/292", + "https://github.com/mitsuhiko/jinja2/pull/296", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-82.yaml" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-0012" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-217", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-217", + "title": "PYSEC-2019-217", + "description": "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1152", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1329", + "https://github.com/advisories/GHSA-462w-v97r-4m45", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml", + "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/", + "https://palletsprojects.com/blog/jinja-2-10-1-released", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ], + "disclosedAt": 1554596940, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-220", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-220", + "title": "PYSEC-2019-220", + "description": "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1022", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1260", + "https://access.redhat.com/errata/RHSA-2019:3964", + "https://access.redhat.com/errata/RHSA-2019:4062", + "https://github.com/advisories/GHSA-hj2j-77xm-mc5v", + "https://github.com/pallets/jinja", + "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-220.yaml", + "https://palletsprojects.com/blog/jinja-281-released/", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2016-10745", + "GHSA-hj2j-77xm-mc5v" + ], + "disclosedAt": 1554730140, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2021-66", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2021-66", + "title": "PYSEC-2021-66", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDoS vulnerability is mainly due to the `_punctuation_re regex` operator and its use of multiple wildcards. The last wildcard is the most exploitable as it searches for trailing punctuation. This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2021-66.yaml", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2020-28493", + "SNYK-PYTHON-JINJA2-1012994", + "GHSA-g3rq-g295-4j3m" + ], + "disclosedAt": 1612210500, + "observedAt": 1645597812 + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/advisories_metadata.txt b/vulntotal/tests/test_data/deps/advisories_metadata.txt new file mode 100644 index 000000000..047b084d2 --- /dev/null +++ b/vulntotal/tests/test_data/deps/advisories_metadata.txt @@ -0,0 +1,208 @@ +{ + "package": { + "system": "PYPI", + "name": "jinja2" + }, + "version": "2.4.1", + "dependencyCount": 0, + "directCount": 0, + "indirectCount": 0, + "dependencies": [ + { + "package": { + "system": "PYPI", + "name": "jinja2" + }, + "version": "2.4.1", + "type": "", + "createdAt": 1271771625, + "description": "A small but fast and easy to use stand-alone template engine written in pure python.", + "owners": [], + "licenses": [ + "non-standard" + ], + "advisories": [ + { + "source": "GHSA", + "sourceID": "GHSA-g3rq-g295-4j3m", + "sourceURL": "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "title": "Regular Expression Denial of Service (ReDoS) in Jinja2", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDOS vulnerability of the regex is mainly due to the sub-pattern [a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+ This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://nvd.nist.gov/vuln/detail/CVE-2020-28493", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://security.gentoo.org/glsa/202107-19", + "https://github.com/advisories/GHSA-g3rq-g295-4j3m" + ], + "severity": "MEDIUM", + "gitHubSeverity": "MODERATE", + "scoreV3": 5.3, + "aliases": [ + "CVE-2020-28493" + ], + "disclosedAt": 1616189285, + "observedAt": 1650328213 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-8", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-8", + "title": "PYSEC-2014-8", + "description": "The default configuration for bccache.FileSystemBytecodeCache in Jinja2 before 2.7.2 does not properly create temporary files, which allows local users to gain privileges via a crafted .cache file with a name starting with __jinja2_ in /tmp.", + "referenceURLs": [ + "http://advisories.mageia.org/MGASA-2014-0028.html", + "http://jinja.pocoo.org/docs/changelog/", + "http://openwall.com/lists/oss-security/2014/01/10/2", + "http://openwall.com/lists/oss-security/2014/01/10/3", + "http://rhn.redhat.com/errata/RHSA-2014-0747.html", + "http://rhn.redhat.com/errata/RHSA-2014-0748.html", + "http://secunia.com/advisories/56287", + "http://secunia.com/advisories/58783", + "http://secunia.com/advisories/58918", + "http://secunia.com/advisories/59017", + "http://secunia.com/advisories/60738", + "http://secunia.com/advisories/60770", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "http://www.mandriva.com/security/advisories?name=MDVSA-2014:096", + "https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-8.yaml", + "https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-1402" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-82", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-82", + "title": "PYSEC-2014-82", + "description": "FileSystemBytecodeCache in Jinja2 2.7.2 does not properly create temporary directories, which allows local users to gain privileges by pre-creating a temporary directory with a user's uid. NOTE: this vulnerability exists because of an incomplete fix for CVE-2014-1402.", + "referenceURLs": [ + "http://seclists.org/oss-sec/2014/q1/73", + "http://secunia.com/advisories/56328", + "http://secunia.com/advisories/60738", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/mitsuhiko/jinja2", + "https://github.com/mitsuhiko/jinja2/commit/acb672b6a179567632e032f547582f30fa2f4aa7", + "https://github.com/mitsuhiko/jinja2/pull/292", + "https://github.com/mitsuhiko/jinja2/pull/296", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-82.yaml" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-0012" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-217", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-217", + "title": "PYSEC-2019-217", + "description": "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1152", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1329", + "https://github.com/advisories/GHSA-462w-v97r-4m45", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml", + "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/", + "https://palletsprojects.com/blog/jinja-2-10-1-released", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ], + "disclosedAt": 1554596940, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-220", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-220", + "title": "PYSEC-2019-220", + "description": "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1022", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1260", + "https://access.redhat.com/errata/RHSA-2019:3964", + "https://access.redhat.com/errata/RHSA-2019:4062", + "https://github.com/advisories/GHSA-hj2j-77xm-mc5v", + "https://github.com/pallets/jinja", + "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-220.yaml", + "https://palletsprojects.com/blog/jinja-281-released/", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2016-10745", + "GHSA-hj2j-77xm-mc5v" + ], + "disclosedAt": 1554730140, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2021-66", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2021-66", + "title": "PYSEC-2021-66", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDoS vulnerability is mainly due to the `_punctuation_re regex` operator and its use of multiple wildcards. The last wildcard is the most exploitable as it searches for trailing punctuation. This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2021-66.yaml", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2020-28493", + "SNYK-PYTHON-JINJA2-1012994", + "GHSA-g3rq-g295-4j3m" + ], + "disclosedAt": 1612210500, + "observedAt": 1645597812 + } + ], + "distance": 0, + "dependencyCount": 0, + "errors": [] + } + ] +} \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/advisory.json b/vulntotal/tests/test_data/deps/advisory.json new file mode 100644 index 000000000..907156046 --- /dev/null +++ b/vulntotal/tests/test_data/deps/advisory.json @@ -0,0 +1,247 @@ +{ + "source": "OSV", + "sourceID": "PYSEC-2019-217", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-217", + "title": "PYSEC-2019-217", + "description": "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1152", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1329", + "https://github.com/advisories/GHSA-462w-v97r-4m45", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml", + "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/", + "https://palletsprojects.com/blog/jinja-2-10-1-released", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ], + "disclosedAt": 1554596940, + "observedAt": 1645597812, + "affectedPackageCount": 734, + "fixedPackageCount": 343, + "ecosystemAffected": 0.001987420225656133, + "packages": [ + { + "package": { + "system": "PyPI", + "name": "jinja2" + }, + "affected": "{[0.0.0.dev0:2.10.1)}", + "rawAffected": "Introduced: 0, Fixed: 2.10.1", + "versionsAffected": [ + { + "version": "2.10", + "createdAt": 1510172022 + }, + { + "version": "2.9.6", + "createdAt": 1491225087 + }, + { + "version": "2.9.5", + "createdAt": 1485614558 + }, + { + "version": "2.9.4", + "createdAt": 1484036657 + }, + { + "version": "2.9.3", + "createdAt": 1483910640 + }, + { + "version": "2.9.2", + "createdAt": 1483840248 + }, + { + "version": "2.9.1", + "createdAt": 1483804587 + }, + { + "version": "2.9", + "createdAt": 1483803350 + }, + { + "version": "2.8.1", + "createdAt": 1483017384 + }, + { + "version": "2.8", + "createdAt": 1437933126 + }, + { + "version": "2.7.3", + "createdAt": 1402073357 + }, + { + "version": "2.7.2", + "createdAt": 1389350560 + }, + { + "version": "2.7.1", + "createdAt": 1375882058 + }, + { + "version": "2.7", + "createdAt": 1369054982 + }, + { + "version": "2.6", + "createdAt": 1311544131 + }, + { + "version": "2.5.5", + "createdAt": 1287412660 + }, + { + "version": "2.5.4", + "createdAt": 1287334122 + }, + { + "version": "2.5.3", + "createdAt": 1287330903 + }, + { + "version": "2.5.2", + "createdAt": 1282130120 + }, + { + "version": "2.5.1", + "createdAt": 1282048744 + }, + { + "version": "2.5", + "createdAt": 1275173682 + }, + { + "version": "2.4.1", + "createdAt": 1271771625 + }, + { + "version": "2.4", + "createdAt": 1271120167 + }, + { + "version": "2.3.1", + "createdAt": 1266586731 + }, + { + "version": "2.3", + "createdAt": 1265768375 + }, + { + "version": "2.2.1", + "createdAt": 1252972721 + }, + { + "version": "2.2", + "createdAt": 1252834253 + }, + { + "version": "2.1.1", + "createdAt": 1230226892 + }, + { + "version": "2.1", + "createdAt": 1227443230 + }, + { + "version": "2.0", + "createdAt": 1216330416 + }, + { + "version": "2.0rc1", + "createdAt": 1213030243 + } + ], + "versionsUnaffected": [ + { + "version": "3.1.2", + "createdAt": 1651166485 + }, + { + "version": "3.1.1", + "createdAt": 1648247575 + }, + { + "version": "3.1.0", + "createdAt": 1648131241 + }, + { + "version": "3.0.3", + "createdAt": 1636489647 + }, + { + "version": "3.0.2", + "createdAt": 1633394967 + }, + { + "version": "3.0.1", + "createdAt": 1621370357 + }, + { + "version": "3.0.0", + "createdAt": 1620767296 + }, + { + "version": "3.0.0rc2", + "createdAt": 1619363578 + }, + { + "version": "3.0.0rc1", + "createdAt": 1618586722 + }, + { + "version": "3.0.0a1", + "createdAt": 1580942178 + }, + { + "version": "2.11.3", + "createdAt": 1612110787 + }, + { + "version": "2.11.2", + "createdAt": 1586794604 + }, + { + "version": "2.11.1", + "createdAt": 1580407772 + }, + { + "version": "2.11.0", + "createdAt": 1580149449 + }, + { + "version": "2.10.3", + "createdAt": 1570215157 + }, + { + "version": "2.10.2", + "createdAt": 1570213187 + }, + { + "version": "2.10.1", + "createdAt": 1554577199 + } + ] + } + ] +} \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/parse_advisory-expected.json b/vulntotal/tests/test_data/deps/parse_advisory-expected.json new file mode 100644 index 000000000..4494c2c2d --- /dev/null +++ b/vulntotal/tests/test_data/deps/parse_advisory-expected.json @@ -0,0 +1,60 @@ +[ + { + "affected_versions": [ + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.10", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1", + "2.7.2", + "2.7.3", + "2.8", + "2.8.1", + "2.9", + "2.9.1", + "2.9.2", + "2.9.3", + "2.9.4", + "2.9.5", + "2.9.6" + ], + "fixed_versions": [ + "2.10.1", + "2.10.2", + "2.10.3", + "2.11.0", + "2.11.1", + "2.11.2", + "2.11.3", + "3.0.0", + "3.0.0a1", + "3.0.0rc1", + "3.0.0rc2", + "3.0.1", + "3.0.2", + "3.0.3", + "3.1.0", + "3.1.1", + "3.1.2" + ], + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/parsed_advisories_metadata-expected.json b/vulntotal/tests/test_data/deps/parsed_advisories_metadata-expected.json new file mode 100644 index 000000000..8f0783ed0 --- /dev/null +++ b/vulntotal/tests/test_data/deps/parsed_advisories_metadata-expected.json @@ -0,0 +1,179 @@ +[ + { + "source": "GHSA", + "sourceID": "GHSA-g3rq-g295-4j3m", + "sourceURL": "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "title": "Regular Expression Denial of Service (ReDoS) in Jinja2", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDOS vulnerability of the regex is mainly due to the sub-pattern [a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+ This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://nvd.nist.gov/vuln/detail/CVE-2020-28493", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://security.gentoo.org/glsa/202107-19", + "https://github.com/advisories/GHSA-g3rq-g295-4j3m" + ], + "severity": "MEDIUM", + "gitHubSeverity": "MODERATE", + "scoreV3": 5.3, + "aliases": [ + "CVE-2020-28493" + ], + "disclosedAt": 1616189285, + "observedAt": 1650328213 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-8", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-8", + "title": "PYSEC-2014-8", + "description": "The default configuration for bccache.FileSystemBytecodeCache in Jinja2 before 2.7.2 does not properly create temporary files, which allows local users to gain privileges via a crafted .cache file with a name starting with __jinja2_ in /tmp.", + "referenceURLs": [ + "http://advisories.mageia.org/MGASA-2014-0028.html", + "http://jinja.pocoo.org/docs/changelog/", + "http://openwall.com/lists/oss-security/2014/01/10/2", + "http://openwall.com/lists/oss-security/2014/01/10/3", + "http://rhn.redhat.com/errata/RHSA-2014-0747.html", + "http://rhn.redhat.com/errata/RHSA-2014-0748.html", + "http://secunia.com/advisories/56287", + "http://secunia.com/advisories/58783", + "http://secunia.com/advisories/58918", + "http://secunia.com/advisories/59017", + "http://secunia.com/advisories/60738", + "http://secunia.com/advisories/60770", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "http://www.mandriva.com/security/advisories?name=MDVSA-2014:096", + "https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-8.yaml", + "https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-1402" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2014-82", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-82", + "title": "PYSEC-2014-82", + "description": "FileSystemBytecodeCache in Jinja2 2.7.2 does not properly create temporary directories, which allows local users to gain privileges by pre-creating a temporary directory with a user's uid. NOTE: this vulnerability exists because of an incomplete fix for CVE-2014-1402.", + "referenceURLs": [ + "http://seclists.org/oss-sec/2014/q1/73", + "http://secunia.com/advisories/56328", + "http://secunia.com/advisories/60738", + "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml", + "https://bugzilla.redhat.com/show_bug.cgi?id=1051421", + "https://github.com/mitsuhiko/jinja2", + "https://github.com/mitsuhiko/jinja2/commit/acb672b6a179567632e032f547582f30fa2f4aa7", + "https://github.com/mitsuhiko/jinja2/pull/292", + "https://github.com/mitsuhiko/jinja2/pull/296", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-82.yaml" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2014-0012" + ], + "disclosedAt": 1400511300, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-217", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-217", + "title": "PYSEC-2019-217", + "description": "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1152", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1329", + "https://github.com/advisories/GHSA-462w-v97r-4m45", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml", + "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E", + "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E", + "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/", + "https://palletsprojects.com/blog/jinja-2-10-1-released", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ], + "disclosedAt": 1554596940, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2019-220", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-220", + "title": "PYSEC-2019-220", + "description": "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape.", + "referenceURLs": [ + "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html", + "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html", + "https://access.redhat.com/errata/RHSA-2019:1022", + "https://access.redhat.com/errata/RHSA-2019:1237", + "https://access.redhat.com/errata/RHSA-2019:1260", + "https://access.redhat.com/errata/RHSA-2019:3964", + "https://access.redhat.com/errata/RHSA-2019:4062", + "https://github.com/advisories/GHSA-hj2j-77xm-mc5v", + "https://github.com/pallets/jinja", + "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-220.yaml", + "https://palletsprojects.com/blog/jinja-281-released/", + "https://usn.ubuntu.com/4011-1/", + "https://usn.ubuntu.com/4011-2/" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2016-10745", + "GHSA-hj2j-77xm-mc5v" + ], + "disclosedAt": 1554730140, + "observedAt": 1645597812 + }, + { + "source": "OSV", + "sourceID": "PYSEC-2021-66", + "sourceURL": "https://osv.dev/vulnerability/PYSEC-2021-66", + "title": "PYSEC-2021-66", + "description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDoS vulnerability is mainly due to the `_punctuation_re regex` operator and its use of multiple wildcards. The last wildcard is the most exploitable as it searches for trailing punctuation. This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.", + "referenceURLs": [ + "https://github.com/advisories/GHSA-g3rq-g295-4j3m", + "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20", + "https://github.com/pallets/jinja/pull/1343", + "https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2021-66.yaml", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/", + "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + ], + "severity": "UNKNOWN", + "gitHubSeverity": "UNKNOWN", + "aliases": [ + "CVE-2020-28493", + "SNYK-PYTHON-JINJA2-1012994", + "GHSA-g3rq-g295-4j3m" + ], + "disclosedAt": 1612210500, + "observedAt": 1645597812 + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/payloads_advisories-expected.json b/vulntotal/tests/test_data/deps/payloads_advisories-expected.json new file mode 100644 index 000000000..38d8e8693 --- /dev/null +++ b/vulntotal/tests/test_data/deps/payloads_advisories-expected.json @@ -0,0 +1,8 @@ +[ + "https://deps.dev/_/advisory/GHSA/GHSA-g3rq-g295-4j3m", + "https://deps.dev/_/advisory/OSV/PYSEC-2014-8", + "https://deps.dev/_/advisory/OSV/PYSEC-2014-82", + "https://deps.dev/_/advisory/OSV/PYSEC-2019-217", + "https://deps.dev/_/advisory/OSV/PYSEC-2019-220", + "https://deps.dev/_/advisory/OSV/PYSEC-2021-66" +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/payloads_meta-expected.json b/vulntotal/tests/test_data/deps/payloads_meta-expected.json new file mode 100644 index 000000000..35262cd90 --- /dev/null +++ b/vulntotal/tests/test_data/deps/payloads_meta-expected.json @@ -0,0 +1,7 @@ +[ + "https://deps.dev/_/s/pypi/p/jinja2/v/2.4.1/dependencies", + "https://deps.dev/_/s/maven/p/org.apache.tomcat%3Atomcat/v/10.1.0-M8/dependencies", + "https://deps.dev/_/s/npm/p/semver-regex/v/3.1.3/dependencies", + "https://deps.dev/_/s/go/p/github.com%2Fcloudflare%2Fcfrpki/v/v1.4.1/dependencies", + "https://deps.dev/_/s/cargo/p/rand/v/0.5.4/dependencies" +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/deps/purls.txt b/vulntotal/tests/test_data/deps/purls.txt new file mode 100644 index 000000000..e2dc92d60 --- /dev/null +++ b/vulntotal/tests/test_data/deps/purls.txt @@ -0,0 +1,5 @@ +pkg:pypi/jinja2@2.4.1 +pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8 +pkg:npm/semver-regex@3.1.3 +pkg:golang/github.com/cloudflare/cfrpki@1.4.1 +pkg:cargo/rand@0.5.4 \ No newline at end of file diff --git a/vulntotal/tests/test_deps.py b/vulntotal/tests/test_deps.py new file mode 100644 index 000000000..a64270061 --- /dev/null +++ b/vulntotal/tests/test_deps.py @@ -0,0 +1,69 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import deps + + +class TestDeps(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "deps") + + def test_generate_meta_payload(self): + file_purls = self.get_test_loc("purls.txt") + with open(file_purls) as f: + purls = f.readlines() + results = [deps.generate_meta_payload(PackageURL.from_string(purl)) for purl in purls] + expected_file = self.get_test_loc("payloads_meta-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_advisory_from_meta(self): + file = self.get_test_loc("advisories_metadata.txt") + with open(file) as f: + metadata = json.load(f) + results = deps.parse_advisories_from_meta(metadata) + expected_file = self.get_test_loc( + "parsed_advisories_metadata-expected.json", must_exist=False + ) + util_tests.check_results_against_json(results, expected_file) + + def test_generate_advisory_payload(self): + file = self.get_test_loc("advisories_metadata.json") + with open(file) as f: + advisories = json.load(f) + results = [deps.generate_advisory_payload(adv) for adv in advisories] + expected_file = self.get_test_loc("payloads_advisories-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_advisory(self): + advisory_file = self.get_test_loc("advisory.json") + with open(advisory_file) as f: + advisory = json.load(f) + results = [adv.to_dict() for adv in deps.parse_advisory(advisory)] + expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From 1724d91ff15a0dbbfd88d07a207c79e02bdec0b8 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:16:16 +0530 Subject: [PATCH 08/36] add Github Datasource - closes: https://github.com/nexB/vulnerablecode/issues/791 Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/github.py | 158 ++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 vulntotal/datasources/github.py diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py new file mode 100644 index 000000000..a708ebce1 --- /dev/null +++ b/vulntotal/datasources/github.py @@ -0,0 +1,158 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + + +import logging +from typing import Iterable + +from packageurl import PackageURL + +from vulnerabilities import utils +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData +from vulntotal.vulntotal_utils import github_constraints_satisfied + +logger = logging.getLogger(__name__) + + +class GithubDataSource(DataSource): + spdx_license_expression = "TODO" + license_url = "TODO" + + def fetch_github(self, graphql_query): + return utils.fetch_github_graphql_query(graphql_query) + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + end_cursor = "" + interesting_edges = [] + while True: + queryset = generate_graphql_payload(purl, end_cursor) + response = self.fetch_github(queryset) + self._raw_dump.append(response) + security_advisories = response["data"]["securityVulnerabilities"] + interesting_edges.extend(extract_interesting_edge(security_advisories["edges"], purl)) + end_cursor = security_advisories["pageInfo"]["endCursor"] + if not security_advisories["pageInfo"]["hasNextPage"]: + break + return parse_advisory(interesting_edges) + + @classmethod + def supported_ecosystem(cls): + return { + "maven": "MAVEN", + "nuget": "NUGET", + "composer": "COMPOSER", + "pypi": "PIP", + "gem": "RUBYGEMS", + "golang": "GO", + "rust": "RUST", + "npm": "NPM", + "erlang": "ERLANG", + } + + +def parse_advisory(interesting_edges) -> Iterable[VendorData]: + for edge in interesting_edges: + aliases = [aliase["value"] for aliase in edge["node"]["advisory"]["identifiers"]] + affected_versions = ( + edge["node"]["vulnerableVersionRange"].strip().replace(" ", "").split(",") + ) + fixed_versions = [edge["node"]["firstPatchedVersion"]["identifier"]] + yield VendorData( + aliases=sorted(list(set(aliases))), + affected_versions=sorted(list(set(affected_versions))), + fixed_versions=sorted(list(set(fixed_versions))), + ) + + +def extract_interesting_edge(edges, purl): + interesting_edges = [] + for edge in edges: + if github_constraints_satisfied(edge["node"]["vulnerableVersionRange"], purl.version): + interesting_edges.append(edge) + return interesting_edges + + +def generate_graphql_payload(purl, end_cursor): + GRAPHQL_QUERY_TEMPLATE = """ + query{ + securityVulnerabilities(first: 100, ecosystem: %s, package: "%s", %s){ + edges { + node { + advisory { + identifiers { + type + value + } + summary + references { + url + } + severity + publishedAt + } + firstPatchedVersion{ + identifier + } + package { + name + } + vulnerableVersionRange + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ + + supported_ecosystem = GithubDataSource.supported_ecosystem() + + if purl.type not in supported_ecosystem: + return + + end_cursor_exp = "" + ecosystem = supported_ecosystem[purl.type] + package_name = purl.name + + if end_cursor: + end_cursor_exp = f'after: "{end_cursor}"' + + if purl.type == "maven": + if not purl.namespace: + logger.error(f"Invalid Maven PURL {str(purl)}") + return + package_name = f"{purl.namespace}:{purl.name}" + + elif purl.type == "composer": + if not purl.namespace: + logger.error(f"Invalid Composer PURL {str(purl)}") + return + package_name = f"{purl.namespace}/{purl.name}" + + elif purl.type == "golang" and purl.namespace: + package_name = f"{purl.namespace}/{purl.name}" + + return {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, package_name, end_cursor_exp)} From 8d42af1985f06b279922bb51971f90c85a8c5ce2 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:21:42 +0530 Subject: [PATCH 09/36] add vulntotal utils Signed-off-by: Keshav Priyadarshi --- vulntotal/vulntotal_utils.py | 134 +++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 vulntotal/vulntotal_utils.py diff --git a/vulntotal/vulntotal_utils.py b/vulntotal/vulntotal_utils.py new file mode 100644 index 000000000..4a8d5670f --- /dev/null +++ b/vulntotal/vulntotal_utils.py @@ -0,0 +1,134 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import operator + + +class GenericVersion: + def __init__(self, version): + self.value = version.replace(" ", "").lstrip("v") + + self.decomposed = tuple( + [int(com) if com.isnumeric() else com for com in self.value.split(".")] + ) + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__eq__(other.value) + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + for i, j in zip(self.decomposed, other.decomposed): + if not isinstance(i, type(j)): + continue + if i.__gt__(j): + return False + return True + + def __le__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.__lt__(other) or self.__eq__(other) + + +def compare(version, package_comparator, package_version): + operator_comparator = { + "<": operator.lt, + ">": operator.gt, + "=": operator.eq, + "<=": operator.le, + ">=": operator.ge, + "==": operator.eq, + "!=": operator.ne, + ")": operator.lt, + "]": operator.le, + "(": operator.gt, + "[": operator.ge, + } + compare = operator_comparator[package_comparator] + return compare(version, package_version) + + +def parse_constraint(constraint): + if constraint.startswith(("<=", ">=", "==", "!=")): + return constraint[:2], constraint[2:] + + if constraint.startswith(("<", ">", "=", "[", "]", "(", ")")): + return constraint[0], constraint[1:] + + if constraint.endswith(("[", "]", "(", ")")): + return constraint[-1], constraint[:-1] + + +def github_constraints_satisfied(github_constrain, version): + gh_constraints = github_constrain.strip().replace(" ", "") + constraints = gh_constraints.split(",") + for constraint in constraints: + gh_comparator, gh_version = parse_constraint(constraint) + if not gh_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare(GenericVersion(version), gh_comparator, GenericVersion(gh_version)): + return False + return True + + +def snky_constraints_satisfied(snyk_constrain, version): + snyk_constraints = snyk_constrain.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + for constraint in constraints: + snyk_comparator, snyk_version = parse_constraint(constraint) + if not snyk_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers or maybe not if snyk is normalizing versions to semver + if not compare(GenericVersion(version), snyk_comparator, GenericVersion(snyk_version)): + return False + return True + + +def gitlab_constraints_satisfied(gitlab_constrain, version): + gitlab_constraints = gitlab_constrain.strip() + constraints = gitlab_constraints.split("||") + + for constraint in constraints: + is_constraint_satisfied = True + + for subcontraint in constraint.strip().split(" "): + + gitlab_comparator, gitlab_version = parse_constraint(subcontraint.strip()) + if not gitlab_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare( + GenericVersion(version), gitlab_comparator, GenericVersion(gitlab_version) + ): + is_constraint_satisfied = False + break + + if is_constraint_satisfied: + return True From d1c7d1711d5bfd8f27f584a4a603dc9ff4364f25 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:22:12 +0530 Subject: [PATCH 10/36] add tests for Github Datasource Signed-off-by: Keshav Priyadarshi --- .../tests/test_data/github/all_edges.json | 275 ++++++++++++++++++ .../extracted_interesting_edge-expected.json | 273 +++++++++++++++++ .../github/graphql_payload-expected.json | 29 ++ .../test_data/github/interesting_edge.json | 273 +++++++++++++++++ .../github/parse_advisory-expected.json | 50 ++++ vulntotal/tests/test_data/github/purls.txt | 9 + vulntotal/tests/test_github.py | 65 +++++ 7 files changed, 974 insertions(+) create mode 100644 vulntotal/tests/test_data/github/all_edges.json create mode 100644 vulntotal/tests/test_data/github/extracted_interesting_edge-expected.json create mode 100644 vulntotal/tests/test_data/github/graphql_payload-expected.json create mode 100644 vulntotal/tests/test_data/github/interesting_edge.json create mode 100644 vulntotal/tests/test_data/github/parse_advisory-expected.json create mode 100644 vulntotal/tests/test_data/github/purls.txt create mode 100644 vulntotal/tests/test_github.py diff --git a/vulntotal/tests/test_data/github/all_edges.json b/vulntotal/tests/test_data/github/all_edges.json new file mode 100644 index 000000000..ee6ad0865 --- /dev/null +++ b/vulntotal/tests/test_data/github/all_edges.json @@ -0,0 +1,275 @@ +{ + "edges":[ + { + "node":{ + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-8r7q-cvjq-x353" + }, + { + "type":"CVE", + "value":"CVE-2014-1402" + } + ], + "summary":"Incorrect Privilege Assignment in Jinja2", + "references":[ + { + "url":"https://nvd.nist.gov/vuln/detail/CVE-2014-1402" + }, + { + "url":"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747" + }, + { + "url":"https://bugzilla.redhat.com/show_bug.cgi?id=1051421" + }, + { + "url":"https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + }, + { + "url":"http://advisories.mageia.org/MGASA-2014-0028.html" + }, + { + "url":"http://jinja.pocoo.org/docs/changelog/" + }, + { + "url":"http://openwall.com/lists/oss-security/2014/01/10/2" + }, + { + "url":"http://openwall.com/lists/oss-security/2014/01/10/3" + }, + { + "url":"http://rhn.redhat.com/errata/RHSA-2014-0747.html" + }, + { + "url":"http://rhn.redhat.com/errata/RHSA-2014-0748.html" + }, + { + "url":"http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml" + }, + { + "url":"https://github.com/advisories/GHSA-8r7q-cvjq-x353" + } + ], + "severity":"MODERATE", + "publishedAt":"2022-05-14T04:04:14Z" + }, + "firstPatchedVersion":{ + "identifier":"2.7.2" + }, + "package":{ + "name":"Jinja2" + }, + "vulnerableVersionRange":"< 2.7.2" + } + }, + { + "node":{ + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-g3rq-g295-4j3m" + }, + { + "type":"CVE", + "value":"CVE-2020-28493" + } + ], + "summary":"Regular Expression Denial of Service (ReDoS) in Jinja2", + "references":[ + { + "url":"https://nvd.nist.gov/vuln/detail/CVE-2020-28493" + }, + { + "url":"https://github.com/pallets/jinja/pull/1343" + }, + { + "url":"https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20" + }, + { + "url":"https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + }, + { + "url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/" + }, + { + "url":"https://security.gentoo.org/glsa/202107-19" + }, + { + "url":"https://github.com/advisories/GHSA-g3rq-g295-4j3m" + } + ], + "severity":"MODERATE", + "publishedAt":"2021-03-19T21:28:05Z" + }, + "firstPatchedVersion":{ + "identifier":"2.11.3" + }, + "package":{ + "name":"jinja2" + }, + "vulnerableVersionRange":"< 2.11.3" + } + }, + { + "node":{ + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-hj2j-77xm-mc5v" + }, + { + "type":"CVE", + "value":"CVE-2016-10745" + } + ], + "summary":"High severity vulnerability that affects Jinja2", + "references":[ + { + "url":"https://nvd.nist.gov/vuln/detail/CVE-2016-10745" + }, + { + "url":"https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1022" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1260" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:3964" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:4062" + }, + { + "url":"https://github.com/advisories/GHSA-hj2j-77xm-mc5v" + }, + { + "url":"https://palletsprojects.com/blog/jinja-281-released/" + }, + { + "url":"https://usn.ubuntu.com/4011-1/" + }, + { + "url":"https://usn.ubuntu.com/4011-2/" + }, + { + "url":"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url":"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity":"HIGH", + "publishedAt":"2019-04-10T14:30:13Z" + }, + "firstPatchedVersion":{ + "identifier":"2.8.1" + }, + "package":{ + "name":"Jinja2" + }, + "vulnerableVersionRange":"< 2.8.1" + } + }, + { + "node":{ + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-462w-v97r-4m45" + }, + { + "type":"CVE", + "value":"CVE-2019-10906" + } + ], + "summary":"High severity vulnerability that affects Jinja2", + "references":[ + { + "url":"https://nvd.nist.gov/vuln/detail/CVE-2019-10906" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1152" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url":"https://access.redhat.com/errata/RHSA-2019:1329" + }, + { + "url":"https://github.com/advisories/GHSA-462w-v97r-4m45" + }, + { + "url":"https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E" + }, + { + "url":"https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E" + }, + { + "url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/" + }, + { + "url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/" + }, + { + "url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/" + }, + { + "url":"https://palletsprojects.com/blog/jinja-2-10-1-released" + }, + { + "url":"https://usn.ubuntu.com/4011-1/" + }, + { + "url":"https://usn.ubuntu.com/4011-2/" + }, + { + "url":"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url":"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity":"HIGH", + "publishedAt":"2019-04-10T14:30:24Z" + }, + "firstPatchedVersion":{ + "identifier":"2.10.1" + }, + "package":{ + "name":"Jinja2" + }, + "vulnerableVersionRange":"< 2.10.1" + } + } + ] +} \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/extracted_interesting_edge-expected.json b/vulntotal/tests/test_data/github/extracted_interesting_edge-expected.json new file mode 100644 index 000000000..e32cac95e --- /dev/null +++ b/vulntotal/tests/test_data/github/extracted_interesting_edge-expected.json @@ -0,0 +1,273 @@ +[ + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-8r7q-cvjq-x353" + }, + { + "type": "CVE", + "value": "CVE-2014-1402" + } + ], + "summary": "Incorrect Privilege Assignment in Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2014-1402" + }, + { + "url": "https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747" + }, + { + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=1051421" + }, + { + "url": "https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + }, + { + "url": "http://advisories.mageia.org/MGASA-2014-0028.html" + }, + { + "url": "http://jinja.pocoo.org/docs/changelog/" + }, + { + "url": "http://openwall.com/lists/oss-security/2014/01/10/2" + }, + { + "url": "http://openwall.com/lists/oss-security/2014/01/10/3" + }, + { + "url": "http://rhn.redhat.com/errata/RHSA-2014-0747.html" + }, + { + "url": "http://rhn.redhat.com/errata/RHSA-2014-0748.html" + }, + { + "url": "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml" + }, + { + "url": "https://github.com/advisories/GHSA-8r7q-cvjq-x353" + } + ], + "severity": "MODERATE", + "publishedAt": "2022-05-14T04:04:14Z" + }, + "firstPatchedVersion": { + "identifier": "2.7.2" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.7.2" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-g3rq-g295-4j3m" + }, + { + "type": "CVE", + "value": "CVE-2020-28493" + } + ], + "summary": "Regular Expression Denial of Service (ReDoS) in Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2020-28493" + }, + { + "url": "https://github.com/pallets/jinja/pull/1343" + }, + { + "url": "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20" + }, + { + "url": "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/" + }, + { + "url": "https://security.gentoo.org/glsa/202107-19" + }, + { + "url": "https://github.com/advisories/GHSA-g3rq-g295-4j3m" + } + ], + "severity": "MODERATE", + "publishedAt": "2021-03-19T21:28:05Z" + }, + "firstPatchedVersion": { + "identifier": "2.11.3" + }, + "package": { + "name": "jinja2" + }, + "vulnerableVersionRange": "< 2.11.3" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-hj2j-77xm-mc5v" + }, + { + "type": "CVE", + "value": "CVE-2016-10745" + } + ], + "summary": "High severity vulnerability that affects Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2016-10745" + }, + { + "url": "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1022" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1260" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:3964" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:4062" + }, + { + "url": "https://github.com/advisories/GHSA-hj2j-77xm-mc5v" + }, + { + "url": "https://palletsprojects.com/blog/jinja-281-released/" + }, + { + "url": "https://usn.ubuntu.com/4011-1/" + }, + { + "url": "https://usn.ubuntu.com/4011-2/" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity": "HIGH", + "publishedAt": "2019-04-10T14:30:13Z" + }, + "firstPatchedVersion": { + "identifier": "2.8.1" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.8.1" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-462w-v97r-4m45" + }, + { + "type": "CVE", + "value": "CVE-2019-10906" + } + ], + "summary": "High severity vulnerability that affects Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2019-10906" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1152" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1329" + }, + { + "url": "https://github.com/advisories/GHSA-462w-v97r-4m45" + }, + { + "url": "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/" + }, + { + "url": "https://palletsprojects.com/blog/jinja-2-10-1-released" + }, + { + "url": "https://usn.ubuntu.com/4011-1/" + }, + { + "url": "https://usn.ubuntu.com/4011-2/" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity": "HIGH", + "publishedAt": "2019-04-10T14:30:24Z" + }, + "firstPatchedVersion": { + "identifier": "2.10.1" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.10.1" + } + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/graphql_payload-expected.json b/vulntotal/tests/test_data/github/graphql_payload-expected.json new file mode 100644 index 000000000..6203de0d9 --- /dev/null +++ b/vulntotal/tests/test_data/github/graphql_payload-expected.json @@ -0,0 +1,29 @@ +[ + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: PIP, package: \"jinja2\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: MAVEN, package: \"org.apache.tomcat:tomcat\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NUGET, package: \"moment.js\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NPM, package: \"semver-regex\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: GO, package: \"github.com/cloudflare/cfrpki\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: COMPOSER, package: \"symfony/symfony\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUST, package: \"slice-deque\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: ERLANG, package: \"alchemist.vim\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + }, + { + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUBYGEMS, package: \"ftpd\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/interesting_edge.json b/vulntotal/tests/test_data/github/interesting_edge.json new file mode 100644 index 000000000..e32cac95e --- /dev/null +++ b/vulntotal/tests/test_data/github/interesting_edge.json @@ -0,0 +1,273 @@ +[ + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-8r7q-cvjq-x353" + }, + { + "type": "CVE", + "value": "CVE-2014-1402" + } + ], + "summary": "Incorrect Privilege Assignment in Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2014-1402" + }, + { + "url": "https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747" + }, + { + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=1051421" + }, + { + "url": "https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html" + }, + { + "url": "http://advisories.mageia.org/MGASA-2014-0028.html" + }, + { + "url": "http://jinja.pocoo.org/docs/changelog/" + }, + { + "url": "http://openwall.com/lists/oss-security/2014/01/10/2" + }, + { + "url": "http://openwall.com/lists/oss-security/2014/01/10/3" + }, + { + "url": "http://rhn.redhat.com/errata/RHSA-2014-0747.html" + }, + { + "url": "http://rhn.redhat.com/errata/RHSA-2014-0748.html" + }, + { + "url": "http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml" + }, + { + "url": "https://github.com/advisories/GHSA-8r7q-cvjq-x353" + } + ], + "severity": "MODERATE", + "publishedAt": "2022-05-14T04:04:14Z" + }, + "firstPatchedVersion": { + "identifier": "2.7.2" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.7.2" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-g3rq-g295-4j3m" + }, + { + "type": "CVE", + "value": "CVE-2020-28493" + } + ], + "summary": "Regular Expression Denial of Service (ReDoS) in Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2020-28493" + }, + { + "url": "https://github.com/pallets/jinja/pull/1343" + }, + { + "url": "https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20" + }, + { + "url": "https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/" + }, + { + "url": "https://security.gentoo.org/glsa/202107-19" + }, + { + "url": "https://github.com/advisories/GHSA-g3rq-g295-4j3m" + } + ], + "severity": "MODERATE", + "publishedAt": "2021-03-19T21:28:05Z" + }, + "firstPatchedVersion": { + "identifier": "2.11.3" + }, + "package": { + "name": "jinja2" + }, + "vulnerableVersionRange": "< 2.11.3" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-hj2j-77xm-mc5v" + }, + { + "type": "CVE", + "value": "CVE-2016-10745" + } + ], + "summary": "High severity vulnerability that affects Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2016-10745" + }, + { + "url": "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1022" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1260" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:3964" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:4062" + }, + { + "url": "https://github.com/advisories/GHSA-hj2j-77xm-mc5v" + }, + { + "url": "https://palletsprojects.com/blog/jinja-281-released/" + }, + { + "url": "https://usn.ubuntu.com/4011-1/" + }, + { + "url": "https://usn.ubuntu.com/4011-2/" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity": "HIGH", + "publishedAt": "2019-04-10T14:30:13Z" + }, + "firstPatchedVersion": { + "identifier": "2.8.1" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.8.1" + } + }, + { + "node": { + "advisory": { + "identifiers": [ + { + "type": "GHSA", + "value": "GHSA-462w-v97r-4m45" + }, + { + "type": "CVE", + "value": "CVE-2019-10906" + } + ], + "summary": "High severity vulnerability that affects Jinja2", + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2019-10906" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1152" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1237" + }, + { + "url": "https://access.redhat.com/errata/RHSA-2019:1329" + }, + { + "url": "https://github.com/advisories/GHSA-462w-v97r-4m45" + }, + { + "url": "https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/" + }, + { + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/" + }, + { + "url": "https://palletsprojects.com/blog/jinja-2-10-1-released" + }, + { + "url": "https://usn.ubuntu.com/4011-1/" + }, + { + "url": "https://usn.ubuntu.com/4011-2/" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" + }, + { + "url": "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" + } + ], + "severity": "HIGH", + "publishedAt": "2019-04-10T14:30:24Z" + }, + "firstPatchedVersion": { + "identifier": "2.10.1" + }, + "package": { + "name": "Jinja2" + }, + "vulnerableVersionRange": "< 2.10.1" + } + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/parse_advisory-expected.json b/vulntotal/tests/test_data/github/parse_advisory-expected.json new file mode 100644 index 000000000..c335c280b --- /dev/null +++ b/vulntotal/tests/test_data/github/parse_advisory-expected.json @@ -0,0 +1,50 @@ +[ + { + "affected_versions": [ + "<2.7.2" + ], + "fixed_versions": [ + "2.7.2" + ], + "aliases": [ + "CVE-2014-1402", + "GHSA-8r7q-cvjq-x353" + ] + }, + { + "affected_versions": [ + "<2.11.3" + ], + "fixed_versions": [ + "2.11.3" + ], + "aliases": [ + "CVE-2020-28493", + "GHSA-g3rq-g295-4j3m" + ] + }, + { + "affected_versions": [ + "<2.8.1" + ], + "fixed_versions": [ + "2.8.1" + ], + "aliases": [ + "CVE-2016-10745", + "GHSA-hj2j-77xm-mc5v" + ] + }, + { + "affected_versions": [ + "<2.10.1" + ], + "fixed_versions": [ + "2.10.1" + ], + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/purls.txt b/vulntotal/tests/test_data/github/purls.txt new file mode 100644 index 000000000..30af3e48c --- /dev/null +++ b/vulntotal/tests/test_data/github/purls.txt @@ -0,0 +1,9 @@ +pkg:pypi/jinja2@2.4.1 +pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8 +pkg:nuget/moment.js@2.18.0 +pkg:npm/semver-regex@3.1.3 +pkg:golang/github.com/cloudflare/cfrpki@0.1.0 +pkg:composer/symfony/symfony@2.7.1 +pkg:rust/slice-deque@0.1.0 +pkg:erlang/alchemist.vim@1.3.0 +pkg:gem/ftpd@0.0.1 \ No newline at end of file diff --git a/vulntotal/tests/test_github.py b/vulntotal/tests/test_github.py new file mode 100644 index 000000000..1bdc57bff --- /dev/null +++ b/vulntotal/tests/test_github.py @@ -0,0 +1,65 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import github + + +class TestGithub(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "github") + + def test_generate_graphql_payload(self): + file_purls = self.get_test_loc("purls.txt") + with open(file_purls) as f: + purls = f.readlines() + results = [ + github.generate_graphql_payload(PackageURL.from_string(purl), "") for purl in purls + ] + expected_file = self.get_test_loc("graphql_payload-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_extract_interesting_edge(self): + file = self.get_test_loc("all_edges.json") + with open(file) as f: + edges = json.load(f) + results = github.extract_interesting_edge( + edges["edges"], PackageURL.from_string("pkg:pypi/jinja2@2.4.1") + ) + expected_file = self.get_test_loc( + "extracted_interesting_edge-expected.json", must_exist=False + ) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_advisory(self): + advisory_file = self.get_test_loc("interesting_edge.json") + with open(advisory_file) as f: + advisory = json.load(f) + results = [adv.to_dict() for adv in github.parse_advisory(advisory)] + expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From 4a62daa299bdada9ac3ba1efa090340126ce6ed4 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:40:54 +0530 Subject: [PATCH 11/36] add OSS-Index Datasource - closes: https://github.com/nexB/vulnerablecode/issues/820 Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/oss_index.py | 110 +++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 vulntotal/datasources/oss_index.py diff --git a/vulntotal/datasources/oss_index.py b/vulntotal/datasources/oss_index.py new file mode 100644 index 000000000..845e249a9 --- /dev/null +++ b/vulntotal/datasources/oss_index.py @@ -0,0 +1,110 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +import logging +import os +from typing import Iterable + +import requests + +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData + +logger = logging.getLogger(__name__) + + +class OSSDataSource(DataSource): + spdx_license_expression = "TODO" + license_url = "TODO" + api_unauthenticated = "https://ossindex.sonatype.org/api/v3/component-report" + api_authenticated = "https://ossindex.sonatype.org/api/v3/authorized/component-report" + + def fetch_json_response(self, coordinates): + username = os.environ.get("OSS_USERNAME", None) + token = os.environ.get("OSS_TOKEN", None) + auth = None + url = self.api_unauthenticated + if username and token: + auth = (username, token) + url = self.api_authenticated + response = requests.post(url, auth=auth, json={"coordinates": coordinates}) + + if response.status_code == 200: + return response.json() + elif response.status_code == 401: + logger.error("Invalid credentials") + elif response.status_code == 429: + msg = ( + "Too many requests" + if auth + else "Too many requests: add OSS_USERNAME and OSS_TOKEN in .env file" + ) + logger.error(msg) + else: + logger.error(f"unknown status code: {response.status_code} while fetching: {url}") + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + if purl.type not in self.supported_ecosystem(): + logger.error("Unsupported PURL") + return + + response = self.fetch_json_response([str(purl)]) + if response: + self._raw_dump.append(response) + return parse_advisory(response) + + @classmethod + def supported_ecosystem(cls): + return { + "cargo": "cargo", + "cocoapods": "cocoapods", + "composer": "composer", + "conan": "conan", + "conda": "conda", + "cran": "cran", + "golang": "golang", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pypi": "pypi", + "rpm": "rpm", + "gem": "gem", + "swift": "swift", + } + + +def parse_advisory(component) -> Iterable[VendorData]: + response = component[0] + if response["vulnerabilities"]: + for vuln in response["vulnerabilities"]: + aliases = [vuln["id"]] + affected_versions = [] + fixed_versions = [] + if "versionRanges" in vuln: + affected_versions.extend(vuln["versionRanges"]) + yield VendorData( + aliases=aliases, + affected_versions=affected_versions, + fixed_versions=fixed_versions, + ) From e639b5eb292ec4362da9509fbe4125c0eba73f1d Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:44:49 +0530 Subject: [PATCH 12/36] add tests for OSS-Index Datasource Signed-off-by: Keshav Priyadarshi --- .../tests/test_data/oss_index/advisory.json | 143 ++++++++++++++++++ .../oss_index/parse_advisory-expected.json | 65 ++++++++ vulntotal/tests/test_oss_index.py | 43 ++++++ 3 files changed, 251 insertions(+) create mode 100644 vulntotal/tests/test_data/oss_index/advisory.json create mode 100644 vulntotal/tests/test_data/oss_index/parse_advisory-expected.json create mode 100644 vulntotal/tests/test_oss_index.py diff --git a/vulntotal/tests/test_data/oss_index/advisory.json b/vulntotal/tests/test_data/oss_index/advisory.json new file mode 100644 index 000000000..346ee9f77 --- /dev/null +++ b/vulntotal/tests/test_data/oss_index/advisory.json @@ -0,0 +1,143 @@ +[ + { + "coordinates":"pkg:golang/github.com/cloudflare/cfrpki@v1.1.0", + "reference":"https://ossindex.sonatype.org/component/pkg:golang/github.com/cloudflare/cfrpki@v1.1.0?utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "vulnerabilities":[ + { + "id":"CVE-2021-3907", + "displayName":"CVE-2021-3907", + "title":"[CVE-2021-3907] CWE-22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')", + "description":"OctoRPKI does not escape a URI with a filename containing \"..\", this allows a repository to create a file, (ex. rsync://example.org/repo/../../etc/cron.daily/evil.roa), which would then be written to disk outside the base cache folder. This could allow for remote code execution on the host machine OctoRPKI is running on.", + "cvssScore":9.8, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "cwe":"CWE-22", + "cve":"CVE-2021-3907", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3907?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3907", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-cqh2-vc2f-q4fh" + ] + }, + { + "id":"CVE-2021-3761", + "displayName":"CVE-2021-3761", + "title":"[CVE-2021-3761] CWE-787: Out-of-bounds Write", + "description":"Any CA issuer in the RPKI can trick OctoRPKI prior to 1.3.0 into emitting an invalid VRP \"MaxLength\" value, causing RTR sessions to terminate. An attacker can use this to disable RPKI Origin Validation in a victim network (for example AS 13335 - Cloudflare) prior to launching a BGP hijack which during normal operations would be rejected as \"RPKI invalid\". Additionally, in certain deployments RTR session flapping in and of itself also could cause BGP routing churn, causing availability issues.", + "cvssScore":7.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "cwe":"CWE-787", + "cve":"CVE-2021-3761", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3761?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3761", + "https://github.com/cloudflare/cfrpki/pull/90", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-c8xp-8mf3-62h9", + "https://github.com/CVEProject/cvelist/commit/8f3edcdf139aa823e1f17aa15b337ab92436ebf7" + ] + }, + { + "id":"CVE-2021-3908", + "displayName":"CVE-2021-3908", + "title":"[CVE-2021-3908] CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion')", + "description":"OctoRPKI does not limit the depth of a certificate chain, allowing for a CA to create children in an ad-hoc fashion, thereby making tree traversal never end.", + "cvssScore":7.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "cwe":"CWE-400", + "cve":"CVE-2021-3908", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3908?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3908", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-g5gj-9ggf-9vmq" + ] + }, + { + "id":"CVE-2021-3909", + "displayName":"CVE-2021-3909", + "title":"[CVE-2021-3909] CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion')", + "description":"OctoRPKI does not limit the length of a connection, allowing for a slowloris DOS attack to take place which makes OctoRPKI wait forever. Specifically, the repository that OctoRPKI sends HTTP requests to will keep the connection open for a day before a response is returned, but does keep drip feeding new bytes to keep the connection alive.", + "cvssScore":7.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "cwe":"CWE-400", + "cve":"CVE-2021-3909", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3909?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3909", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-8cvr-4rrf-f244" + ] + }, + { + "id":"CVE-2021-3910", + "displayName":"CVE-2021-3910", + "title":"[CVE-2021-3910] CWE-20: Improper Input Validation", + "description":"OctoRPKI crashes when encountering a repository that returns an invalid ROA (just an encoded NUL (\\0) character).", + "cvssScore":7.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "cwe":"CWE-20", + "cve":"CVE-2021-3910", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3910?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3910", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-5mxh-2qfv-4g7j" + ] + }, + { + "id":"CVE-2021-3978", + "displayName":"CVE-2021-3978", + "title":"[CVE-2021-3978] CWE-281: Improper Preservation of Permissions", + "description":"github.com/cloudflare/cfrpki - Improper Preservation of Permissions", + "cvssScore":7.4, + "cvssVector":"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:N", + "cwe":"CWE-281", + "cve":"CVE-2021-3978", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3978?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3978", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-3pqh-p72c-fj85" + ] + }, + { + "id":"CVE-2021-3911", + "displayName":"CVE-2021-3911", + "title":"[CVE-2021-3911] CWE-252: Unchecked Return Value", + "description":"If the ROA that a repository returns contains too many bits for the IP address then OctoRPKI will crash.", + "cvssScore":6.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + "cwe":"CWE-252", + "cve":"CVE-2021-3911", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3911?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3911", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-w6ww-fmfx-2x22" + ] + }, + { + "id":"CVE-2021-3912", + "displayName":"CVE-2021-3912", + "title":"[CVE-2021-3912] CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion')", + "description":"OctoRPKI tries to load the entire contents of a repository in memory, and in the case of a GZIP bomb, unzip it in memory, making it possible to create a repository that makes OctoRPKI run out of memory (and thus crash).", + "cvssScore":6.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H", + "cwe":"CWE-400", + "cve":"CVE-2021-3912", + "reference":"https://ossindex.sonatype.org/vulnerability/CVE-2021-3912?component-type=golang&component-name=github.com%2Fcloudflare%2Fcfrpki&utm_source=python-requests&utm_medium=integration&utm_content=2.27.1", + "externalReferences":[ + "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2021-3912", + "https://github.com/cloudflare/cfrpki/security/advisories/GHSA-g9wh-3vrx-r7hg" + ] + }, + { + "id":"sonatype-2022-0972", + "displayName":"sonatype-2022-0972", + "title":"1 vulnerability found", + "description":"1 non-CVE vulnerability found. To see more details, please create a free account at https://ossindex.sonatype.org/ and request for this information using your registered account", + "cvssScore":6.5, + "cvssVector":"CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "cwe":"CWE-699", + "reference":"https://ossindex.sonatype.org/vulnerability/sonatype-2022-0972", + "externalReferences":[ + + ] + } + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json b/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json new file mode 100644 index 000000000..fa2708551 --- /dev/null +++ b/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json @@ -0,0 +1,65 @@ +[ + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3907" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3761" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3908" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3909" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3910" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3978" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3911" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "CVE-2021-3912" + ] + }, + { + "affected_versions": [], + "fixed_versions": [], + "aliases": [ + "sonatype-2022-0972" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_oss_index.py b/vulntotal/tests/test_oss_index.py new file mode 100644 index 000000000..5c5dd6854 --- /dev/null +++ b/vulntotal/tests/test_oss_index.py @@ -0,0 +1,43 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import oss_index + + +class TestDeps(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "oss_index") + + def test_parse_advisory(self): + advisory_file = self.get_test_loc("advisory.json") + with open(advisory_file) as f: + advisory = json.load(f) + results = [adv.to_dict() for adv in oss_index.parse_advisory(advisory)] + expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From 03fc9b095fbb8b5d1428d96965b744b7bdd12cf1 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:50:59 +0530 Subject: [PATCH 13/36] add VulnerableCode DataSource Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/vulnerablecode.py | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/vulntotal/datasources/vulnerablecode.py b/vulntotal/datasources/vulnerablecode.py index 4a3bf33bb..66b39aa52 100644 --- a/vulntotal/datasources/vulnerablecode.py +++ b/vulntotal/datasources/vulnerablecode.py @@ -20,3 +20,83 @@ # for any legal advice. # VulnTotal is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +import logging +from typing import Iterable +from urllib.parse import urljoin + +import requests +from packageurl import PackageURL + +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData + +logger = logging.getLogger(__name__) + + +class VulnerableCodeDataSource(DataSource): + spdx_license_expression = "CC-BY-SA-4.0" + license_url = "https://github.com/nexB/vulnerablecode/blob/main/cc-by-sa-4.0.LICENSE" + + global_instance = None + vc_purl_search_api_path = "api/packages/bulk_search/" + + def fetch_post_json(self, payload): + vc_instance = self.global_instance if self.global_instance else "http://localhost:8001/" + + url = urljoin(vc_instance, self.vc_purl_search_api_path) + response = requests.post(url, json=payload) + if not response.status_code == 200: + logger.error(f"Error while fetching {url}") + return + return response.json() + + def fetch_get_json(self, url): + response = requests.get(url) + if not response.status_code == 200: + logger.error(f"Error while fetching {url}") + return + return response.json() + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + if purl.type not in self.supported_ecosystem() or not purl.version: + return + metadata_advisories = self.fetch_post_json({"purls": [str(purl)]}) + self._raw_dump.append(metadata_advisories) + if metadata_advisories and "affected_by_vulnerabilities" in metadata_advisories[0]: + for advisory in metadata_advisories[0]["affected_by_vulnerabilities"]: + fetched_advisory = self.fetch_get_json(advisory["url"]) + self._raw_dump.append(fetched_advisory) + yield parse_advisory(fetched_advisory) + + @classmethod + def supported_ecosystem(cls): + return { + "alpine": "alpine", + "cargo": "cargo", + "composer": "composer", + "deb": "deb", + "golang": "golang", + "maven": "maven", + "nginx": "nginx", + "npm": "npm", + "nuget": "nuget", + "pypi": "pypi", + "rpm": "rpm", + "gem": "gem", + "openssl": "openssl", + } + + +def parse_advisory(fetched_advisory) -> VendorData: + aliases = [aliase["alias"] for aliase in fetched_advisory["aliases"]] + affected_versions = [] + fixed_versions = [] + for instance in fetched_advisory["affected_packages"]: + affected_versions.append(PackageURL.from_string(instance["purl"]).version) + for instance in fetched_advisory["fixed_packages"]: + fixed_versions.append(PackageURL.from_string(instance["purl"]).version) + return VendorData( + aliases=aliases, affected_versions=affected_versions, fixed_versions=fixed_versions + ) From 7b1faf8dac8c3f3063151bc81eb56e478bf89124 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:02:03 +0530 Subject: [PATCH 14/36] add Snyk Datasource - closes: https://github.com/nexB/vulnerablecode/issues/834 Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/snyk.py | 178 ++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 vulntotal/datasources/snyk.py diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py new file mode 100644 index 000000000..3783abd05 --- /dev/null +++ b/vulntotal/datasources/snyk.py @@ -0,0 +1,178 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +import logging +from typing import Iterable +from urllib.parse import quote + +import requests +from bs4 import BeautifulSoup +from packageurl import PackageURL + +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData +from vulntotal.vulntotal_utils import snky_constraints_satisfied + +logger = logging.getLogger(__name__) + + +class SnykDataSource(DataSource): + spdx_license_expression = "TODO" + license_url = "TODO" + + def fetch(self, url): + response = requests.get(url) + if not response.status_code == 200: + logger.error(f"Error while fetching {url}") + return + if response.headers["content-type"] == "application/json, charset=utf-8": + return response.json() + return response.text + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + package_advisory_url = generate_package_advisory_url(purl) + package_advisories_list = self.fetch(package_advisory_url) + self._raw_dump.append(package_advisories_list) + if package_advisories_list: + advisories = extract_html_json_advisories(package_advisories_list) + for snyk_id, affected in advisories.items(): + if "*" in affected or is_purl_in_affected(purl.version, affected): + advisory_payload = generate_advisory_payload(snyk_id) + advisory_html = self.fetch(advisory_payload) + self._raw_dump.append(advisory_html) + if advisory_html: + yield parse_html_advisory(advisory_html, snyk_id, affected) + + @classmethod + def supported_ecosystem(cls): + return { + "cocoapods": "cocoapods", + "composer": "composer", + "golang": "golang", + "hex": "hex", + "linux": "linux", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pypi": "pip", + "rubygems": "rubygems", + # any purl.type not in supported_ecosystem shall implicitly be treated as unmanaged type + "unmanaged": "unmanaged", + } + + +def generate_package_advisory_url(purl): + url_package_advisories = "https://security.snyk.io/package/{ecosystem}/{package}" + + # Pseudo API, unfortunately gives only 30 vulnerability per package, but this is the best we have for unmanaged packages + url_unmanaged_package_advisories = ( + "https://security.snyk.io/api/listing?search={package}&type=unmanaged" + ) + supported_ecosystem = SnykDataSource.supported_ecosystem() + + if purl.type == "unmanaged" or purl.type not in supported_ecosystem: + return url_unmanaged_package_advisories.format( + package=purl.name if not purl.namespace else f"{purl.namespace}/{purl.name}", + ) + + purl_name = purl.name + if purl.type == "maven": + if not purl.namespace: + logger.error(f"Invalid Maven PURL {str(purl)}") + return + purl_name = quote(f"{purl.namespace}:{purl.name}", safe="") + + elif purl.type in ("golang", "composer"): + if purl.namespace: + purl_name = quote(f"{purl.namespace}/{purl.name}", safe="") + + elif purl.type == "linux": + distro = purl.qualifiers["distro"] + purl_name = f"{distro}/{purl.name}" + + return url_package_advisories.format( + ecosystem=supported_ecosystem[purl.type], + package=purl_name, + ) + + +def extract_html_json_advisories(package_advisories): + vulnerablity = {} + + # If advisories are json and is obtained through pseudo API + if isinstance(package_advisories, dict): + if package_advisories["status"] == "ok": + for vuln in package_advisories["vulnerabilities"]: + vulnerablity[vuln["id"]] = vuln["semver"]["vulnerable"] + else: + soup = BeautifulSoup(package_advisories, "html.parser") + vulns_table = soup.find("tbody", class_="vue--table__tbody") + if vulns_table: + vulns_rows = vulns_table.find_all("tr", class_="vue--table__row") + for row in vulns_rows: + anchor = row.find(class_="vue--anchor") + ranges = row.find_all( + "span", class_="vue--chip vulnerable-versions__chip vue--chip--default" + ) + affected_versions = [vers.text.strip() for vers in ranges] + vulnerablity[anchor["href"].rsplit("/", 1)[-1]] = affected_versions + return vulnerablity + + +def parse_html_advisory(advisory_html, snyk_id, affected) -> VendorData: + aliases = [] + fixed_versions = [] + + advisory_soup = BeautifulSoup(advisory_html, "html.parser") + cve_span = advisory_soup.find("span", class_="cve") + if cve_span: + cve_anchor = cve_span.find("a", class_="vue--anchor") + aliases.append(cve_anchor["id"]) + + how_to_fix = advisory_soup.find( + "div", class_="vue--block vuln-page__instruction-block vue--block--instruction" + ) + if how_to_fix: + fixed = how_to_fix.find("p").text.split(" ") + if "Upgrade" in fixed: + lower = fixed.index("version") if "version" in fixed else fixed.index("versions") + upper = fixed.index("or") + fixed_versions = "".join(fixed[lower + 1 : upper]).split(",") + aliases.append(snyk_id) + return VendorData( + aliases=aliases, + affected_versions=affected, + fixed_versions=fixed_versions, + ) + + +def is_purl_in_affected(version, affected): + for affected_range in affected: + if snky_constraints_satisfied(affected_range, version): + return True + return False + + +def generate_advisory_payload(snyk_id): + return f"https://security.snyk.io/vuln/{snyk_id}" From 13328c7c3aec43053bb03d10e84a2c3b0d5a1c4f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:03:15 +0530 Subject: [PATCH 15/36] add vulntotal utils Signed-off-by: Keshav Priyadarshi --- vulntotal/vulntotal_utils.py | 134 +++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 vulntotal/vulntotal_utils.py diff --git a/vulntotal/vulntotal_utils.py b/vulntotal/vulntotal_utils.py new file mode 100644 index 000000000..4a8d5670f --- /dev/null +++ b/vulntotal/vulntotal_utils.py @@ -0,0 +1,134 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import operator + + +class GenericVersion: + def __init__(self, version): + self.value = version.replace(" ", "").lstrip("v") + + self.decomposed = tuple( + [int(com) if com.isnumeric() else com for com in self.value.split(".")] + ) + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__eq__(other.value) + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + for i, j in zip(self.decomposed, other.decomposed): + if not isinstance(i, type(j)): + continue + if i.__gt__(j): + return False + return True + + def __le__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.__lt__(other) or self.__eq__(other) + + +def compare(version, package_comparator, package_version): + operator_comparator = { + "<": operator.lt, + ">": operator.gt, + "=": operator.eq, + "<=": operator.le, + ">=": operator.ge, + "==": operator.eq, + "!=": operator.ne, + ")": operator.lt, + "]": operator.le, + "(": operator.gt, + "[": operator.ge, + } + compare = operator_comparator[package_comparator] + return compare(version, package_version) + + +def parse_constraint(constraint): + if constraint.startswith(("<=", ">=", "==", "!=")): + return constraint[:2], constraint[2:] + + if constraint.startswith(("<", ">", "=", "[", "]", "(", ")")): + return constraint[0], constraint[1:] + + if constraint.endswith(("[", "]", "(", ")")): + return constraint[-1], constraint[:-1] + + +def github_constraints_satisfied(github_constrain, version): + gh_constraints = github_constrain.strip().replace(" ", "") + constraints = gh_constraints.split(",") + for constraint in constraints: + gh_comparator, gh_version = parse_constraint(constraint) + if not gh_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare(GenericVersion(version), gh_comparator, GenericVersion(gh_version)): + return False + return True + + +def snky_constraints_satisfied(snyk_constrain, version): + snyk_constraints = snyk_constrain.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + for constraint in constraints: + snyk_comparator, snyk_version = parse_constraint(constraint) + if not snyk_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers or maybe not if snyk is normalizing versions to semver + if not compare(GenericVersion(version), snyk_comparator, GenericVersion(snyk_version)): + return False + return True + + +def gitlab_constraints_satisfied(gitlab_constrain, version): + gitlab_constraints = gitlab_constrain.strip() + constraints = gitlab_constraints.split("||") + + for constraint in constraints: + is_constraint_satisfied = True + + for subcontraint in constraint.strip().split(" "): + + gitlab_comparator, gitlab_version = parse_constraint(subcontraint.strip()) + if not gitlab_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare( + GenericVersion(version), gitlab_comparator, GenericVersion(gitlab_version) + ): + is_constraint_satisfied = False + break + + if is_constraint_satisfied: + return True From 9a38cf214c329130488ab195d4cd9778c3090012 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:09:12 +0530 Subject: [PATCH 16/36] add Gitlab Datasource - closes: https://github.com/nexB/vulnerablecode/issues/836 Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/gitlab.py | 182 ++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 vulntotal/datasources/gitlab.py diff --git a/vulntotal/datasources/gitlab.py b/vulntotal/datasources/gitlab.py new file mode 100644 index 000000000..a82ba6c30 --- /dev/null +++ b/vulntotal/datasources/gitlab.py @@ -0,0 +1,182 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + + +import json +import logging +import os +import shutil +import tarfile +from pathlib import Path +from typing import Iterable + +import requests +import saneyaml +from fetchcode import fetch +from packageurl import PackageURL + +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData +from vulntotal.vulntotal_utils import gitlab_constraints_satisfied + +logger = logging.getLogger(__name__) + + +class GitlabDataSource(DataSource): + spdx_license_expression = "TODO" + license_url = "TODO" + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + package_slug = get_package_slug(purl) + location = download_subtree(package_slug, speculative_execution=True) + if not location: + clear_download(location) + path = self.supported_ecosystem()[purl.type] + casesensitive_package_slug = get_casesensitive_slug(path, package_slug) + location = download_subtree(casesensitive_package_slug) + if location: + interesting_advisories = parse_interesting_advisories(location, purl.version, delete_download=True) + return interesting_advisories + clear_download(location) + + @classmethod + def supported_ecosystem(cls): + return { + "composer": "packagist", + "conan": "conan", + "gem": "gem", + "golang": "go", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pypi": "pypi", + } + + +def get_package_slug(purl): + supported_ecosystem = GitlabDataSource.supported_ecosystem() + + if purl.type not in supported_ecosystem: + return + + ecosystem = supported_ecosystem[purl.type] + package_name = purl.name + + if purl.type in ("maven", "composer", "golang"): + package_name = f"{purl.namespace}/{purl.name}" + + return f"{ecosystem}/{package_name}" + + +def download_subtree(package_slug: str, speculative_execution=False): + url = f"https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/archive/master/gemnasium-db-master.tar.gz?path={package_slug}" + response = fetch(url) + if os.path.getsize(response.location) > 0: + extracted_location = Path(response.location).parent.joinpath( + "temp_vulntotal_gitlab_datasource" + ) + with tarfile.open(response.location, "r") as file_obj: + file_obj.extractall(extracted_location) + os.remove(response.location) + return extracted_location + if not speculative_execution: + logger.error(f"{package_slug} doesn't exist") + os.remove(response.location) + + +def clear_download(location): + if location: + shutil.rmtree(location) + + +def get_casesensitive_slug(path, package_slug): + payload = [ + { + "operationName": "getPaginatedTree", + "variables": { + "projectPath": "gitlab-org/security-products/gemnasium-db", + "ref": "master", + "path": path, + "nextPageCursor": "", + "pageSize": 100, + }, + "query": """ + fragment TreeEntry on Entry { + flatPath + } + query getPaginatedTree($projectPath: ID!, $path: String, $ref: String!, $nextPageCursor: String) { + project(fullPath: $projectPath) { + repository { + paginatedTree(path: $path, ref: $ref, after: $nextPageCursor) { + pageInfo { + endCursor + startCursor + hasNextPage + } + nodes { + trees { + nodes { + ...TreeEntry + } + } + } + } + } + } + } """, + } + ] + url = "https://gitlab.com/api/graphql" + hasnext = True + + while hasnext: + response = requests.post(url, json=payload).json() + paginated_tree = response[0]["data"]["project"]["repository"]["paginatedTree"] + + for slug in paginated_tree["nodes"][0]["trees"]["nodes"]: + if slug["flatPath"].lower() == package_slug.lower(): + return slug["flatPath"] + + # If the namespace/subfolder contains multiple packages, then progressive transverse through folders tree + if package_slug.lower().startswith(slug["flatPath"].lower()): + return get_gitlab_style_slug(slug["flatPath"], package_slug) + + payload[0]["variables"]["nextPageCursor"] = paginated_tree["pageInfo"]["endCursor"] + hasnext = paginated_tree["pageInfo"]["hasNextPage"] + + +def parse_interesting_advisories(location, version, delete_download=False) -> Iterable[VendorData]: + path = Path(location) + glob = "**/*.yml" + files = (p for p in path.glob(glob) if p.is_file()) + for file in sorted(files): + with open(file) as f: + gitlab_advisory = saneyaml.load(f) + if gitlab_constraints_satisfied(gitlab_advisory["affected_range"], version): + yield VendorData( + aliases=gitlab_advisory["identifiers"], + affected_versions=[gitlab_advisory["affected_range"]], + fixed_versions=gitlab_advisory["fixed_versions"], + ) + if delete_download: + clear_download(location) From 0378b6ce87e9b7f9418ad34dbb5dec6528f542bf Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:10:27 +0530 Subject: [PATCH 17/36] add vulntotal utils Signed-off-by: Keshav Priyadarshi --- vulntotal/vulntotal_utils.py | 134 +++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 vulntotal/vulntotal_utils.py diff --git a/vulntotal/vulntotal_utils.py b/vulntotal/vulntotal_utils.py new file mode 100644 index 000000000..4a8d5670f --- /dev/null +++ b/vulntotal/vulntotal_utils.py @@ -0,0 +1,134 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import operator + + +class GenericVersion: + def __init__(self, version): + self.value = version.replace(" ", "").lstrip("v") + + self.decomposed = tuple( + [int(com) if com.isnumeric() else com for com in self.value.split(".")] + ) + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__eq__(other.value) + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + for i, j in zip(self.decomposed, other.decomposed): + if not isinstance(i, type(j)): + continue + if i.__gt__(j): + return False + return True + + def __le__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.__lt__(other) or self.__eq__(other) + + +def compare(version, package_comparator, package_version): + operator_comparator = { + "<": operator.lt, + ">": operator.gt, + "=": operator.eq, + "<=": operator.le, + ">=": operator.ge, + "==": operator.eq, + "!=": operator.ne, + ")": operator.lt, + "]": operator.le, + "(": operator.gt, + "[": operator.ge, + } + compare = operator_comparator[package_comparator] + return compare(version, package_version) + + +def parse_constraint(constraint): + if constraint.startswith(("<=", ">=", "==", "!=")): + return constraint[:2], constraint[2:] + + if constraint.startswith(("<", ">", "=", "[", "]", "(", ")")): + return constraint[0], constraint[1:] + + if constraint.endswith(("[", "]", "(", ")")): + return constraint[-1], constraint[:-1] + + +def github_constraints_satisfied(github_constrain, version): + gh_constraints = github_constrain.strip().replace(" ", "") + constraints = gh_constraints.split(",") + for constraint in constraints: + gh_comparator, gh_version = parse_constraint(constraint) + if not gh_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare(GenericVersion(version), gh_comparator, GenericVersion(gh_version)): + return False + return True + + +def snky_constraints_satisfied(snyk_constrain, version): + snyk_constraints = snyk_constrain.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + for constraint in constraints: + snyk_comparator, snyk_version = parse_constraint(constraint) + if not snyk_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers or maybe not if snyk is normalizing versions to semver + if not compare(GenericVersion(version), snyk_comparator, GenericVersion(snyk_version)): + return False + return True + + +def gitlab_constraints_satisfied(gitlab_constrain, version): + gitlab_constraints = gitlab_constrain.strip() + constraints = gitlab_constraints.split("||") + + for constraint in constraints: + is_constraint_satisfied = True + + for subcontraint in constraint.strip().split(" "): + + gitlab_comparator, gitlab_version = parse_constraint(subcontraint.strip()) + if not gitlab_version: + continue + # TODO: Replace the GenericVersion with ecosystem specific from univers + if not compare( + GenericVersion(version), gitlab_comparator, GenericVersion(gitlab_version) + ): + is_constraint_satisfied = False + break + + if is_constraint_satisfied: + return True From 55fc8a849adf82511bfcb83190469b156e848337 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:11:59 +0530 Subject: [PATCH 18/36] test Gitlab Datasource Signed-off-by: Keshav Priyadarshi --- .../gitlab/package_advisory_url-expected.json | 8 +++ .../gitlab/parsed_advisory-expected.json | 47 ++++++++++++++++ vulntotal/tests/test_data/gitlab/purls.txt | 6 ++ .../pypi/Jinja2/CVE-2014-1402.yml | 28 ++++++++++ .../pypi/Jinja2/CVE-2016-10745.yml | 37 ++++++++++++ .../pypi/Jinja2/CVE-2019-10906.yml | 24 ++++++++ .../pypi/Jinja2/CVE-2019-8341.yml | 28 ++++++++++ .../pypi/Jinja2/CVE-2020-28493.yml | 26 +++++++++ vulntotal/tests/test_gitlab.py | 56 +++++++++++++++++++ 9 files changed, 260 insertions(+) create mode 100644 vulntotal/tests/test_data/gitlab/package_advisory_url-expected.json create mode 100644 vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json create mode 100644 vulntotal/tests/test_data/gitlab/purls.txt create mode 100644 vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2014-1402.yml create mode 100644 vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2016-10745.yml create mode 100644 vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-10906.yml create mode 100644 vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-8341.yml create mode 100644 vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2020-28493.yml create mode 100644 vulntotal/tests/test_gitlab.py diff --git a/vulntotal/tests/test_data/gitlab/package_advisory_url-expected.json b/vulntotal/tests/test_data/gitlab/package_advisory_url-expected.json new file mode 100644 index 000000000..c55362065 --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/package_advisory_url-expected.json @@ -0,0 +1,8 @@ +[ + "pypi/jinja2", + "maven/org.apache.tomcat/tomcat", + "npm/semver-regex", + "go/github.com/mattermost/mattermost-server/v6/api4", + "packagist/bolt/core", + "nuget/moment.js" +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json b/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json new file mode 100644 index 000000000..e3598ecea --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json @@ -0,0 +1,47 @@ +[ + { + "affected_versions": [ + "<=2.7.1" + ], + "fixed_versions": [ + "2.7.2" + ], + "aliases": [ + "CVE-2014-1402" + ] + }, + { + "affected_versions": [ + "<2.8.1" + ], + "fixed_versions": [ + "2.8.1" + ], + "aliases": [ + "GHSA-hj2j-77xm-mc5v", + "CVE-2016-10745" + ] + }, + { + "affected_versions": [ + "<2.10.1" + ], + "fixed_versions": [ + "2.10.1" + ], + "aliases": [ + "CVE-2019-10906" + ] + }, + { + "affected_versions": [ + "<2.11.3" + ], + "fixed_versions": [ + "2.11.3" + ], + "aliases": [ + "CVE-2020-28493" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/gitlab/purls.txt b/vulntotal/tests/test_data/gitlab/purls.txt new file mode 100644 index 000000000..b18b6af70 --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/purls.txt @@ -0,0 +1,6 @@ +pkg:pypi/jinja2@2.4.1 +pkg:maven/org.apache.tomcat/tomcat@10.1.0 +pkg:npm/semver-regex@3.1.3 +pkg:golang/github.com/mattermost/mattermost-server/v6/api4@0.1 +pkg:composer/bolt/core@0.1 +pkg:nuget/moment.js@2.18.0 \ No newline at end of file diff --git a/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2014-1402.yml b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2014-1402.yml new file mode 100644 index 000000000..797936bc7 --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2014-1402.yml @@ -0,0 +1,28 @@ +--- +identifier: "CVE-2014-1402" +package_slug: "pypi/Jinja2" +title: "Incorrect Default Permissions" +description: "The default configuration for `bccache.FileSystemBytecodeCache` in Jinja2 + before does not properly create temporary files, which allows local users to gain + privileges via a crafted `.cache` file with a name starting with `__jinja2_` in + `/tmp`." +date: "2017-12-21" +pubdate: "2014-05-19" +affected_range: "<=2.7.1" +fixed_versions: +- "2.7.2" +affected_versions: "All versions up to 2.7.1" +not_impacted: "All versions after 2.7.1" +solution: "Upgrade to version 2.7.2 or above." +urls: +- "https://bugzilla.redhat.com/CVE-2014-1402" +- "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747" +- "https://github.com/mitsuhiko/jinja2/commit/acb672b6" +cvss_v2: "AV:L/AC:M/Au:N/C:P/I:P/A:P" +uuid: "73933300-77cd-49dd-8e3d-671763767b99" +cwe_ids: +- "CWE-1035" +- "CWE-264" +- "CWE-937" +identifiers: +- "CVE-2014-1402" diff --git a/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2016-10745.yml b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2016-10745.yml new file mode 100644 index 000000000..493ce6d67 --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2016-10745.yml @@ -0,0 +1,37 @@ +--- +identifier: "CVE-2016-10745" +identifiers: +- "GHSA-hj2j-77xm-mc5v" +- "CVE-2016-10745" +package_slug: "pypi/Jinja2" +title: "Use of Externally-Controlled Format String" +description: "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape." +date: "2021-09-14" +pubdate: "2019-04-10" +affected_range: "<2.8.1" +fixed_versions: +- "2.8.1" +affected_versions: "All versions before 2.8.1" +not_impacted: "All versions starting from 2.8.1" +solution: "Upgrade to version 2.8.1 or above." +urls: +- "https://nvd.nist.gov/vuln/detail/CVE-2016-10745" +- "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16" +- "https://access.redhat.com/errata/RHSA-2019:1022" +- "https://access.redhat.com/errata/RHSA-2019:1237" +- "https://access.redhat.com/errata/RHSA-2019:1260" +- "https://access.redhat.com/errata/RHSA-2019:3964" +- "https://access.redhat.com/errata/RHSA-2019:4062" +- "https://github.com/advisories/GHSA-hj2j-77xm-mc5v" +- "https://palletsprojects.com/blog/jinja-281-released/" +- "https://usn.ubuntu.com/4011-1/" +- "https://usn.ubuntu.com/4011-2/" +- "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html" +- "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html" +cvss_v2: "AV:N/AC:L/Au:N/C:P/I:N/A:N" +cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:N/A:N" +uuid: "db196e91-7fc4-4b94-93b3-e0c0dc01bcbe" +cwe_ids: +- "CWE-1035" +- "CWE-134" +- "CWE-937" diff --git a/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-10906.yml b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-10906.yml new file mode 100644 index 000000000..6c582a949 --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-10906.yml @@ -0,0 +1,24 @@ +--- +identifier: "CVE-2019-10906" +package_slug: "pypi/Jinja2" +title: "Sandbox Escape" +description: "In Pallets Jinja, str.format_map allows a sandbox escape." +date: "2019-06-06" +pubdate: "2019-04-06" +affected_range: "<2.10.1" +fixed_versions: +- "2.10.1" +affected_versions: "All versions before 2.10.1" +not_impacted: "All versions starting from 2.10.1" +solution: "Upgrade to version 2.10.1 or above." +urls: +- "https://nvd.nist.gov/vuln/detail/CVE-2019-10906" +- "https://palletsprojects.com/blog/jinja-2-10-1-released" +cvss_v2: "AV:N/AC:L/Au:N/C:P/I:N/A:N" +cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:N/A:N" +uuid: "b2d59abd-b748-4d93-bb21-55f9e9aecea2" +cwe_ids: +- "CWE-1035" +- "CWE-937" +identifiers: +- "CVE-2019-10906" diff --git a/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-8341.yml b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-8341.yml new file mode 100644 index 000000000..cdad470ba --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-8341.yml @@ -0,0 +1,28 @@ +--- +identifier: "CVE-2019-8341" +package_slug: "pypi/Jinja2" +title: "Code Injection" +description: "The `from_string` function is prone to Server Side Template Injection + (SSTI) where it takes the `source` parameter as a template object, renders it, and + then returns it. The attacker can exploit it with `{{INJECTION COMMANDS}}` in a + URI." +date: "2019-08-06" +pubdate: "2019-02-15" +affected_range: "==2.10" +fixed_versions: +- "2.10.1" +affected_versions: "Version 2.10" +not_impacted: "All versions before 2.10, all versions after 2.10" +solution: "Upgrade to version 2.10.1 or above." +urls: +- "https://nvd.nist.gov/vuln/detail/CVE-2019-8341" +- "https://www.exploit-db.com/exploits/46386/" +cvss_v2: "AV:N/AC:L/Au:N/C:P/I:P/A:P" +cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" +uuid: "6b00a76b-aac8-4ce1-b7b2-122a73f22985" +cwe_ids: +- "CWE-1035" +- "CWE-937" +- "CWE-94" +identifiers: +- "CVE-2019-8341" diff --git a/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2020-28493.yml b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2020-28493.yml new file mode 100644 index 000000000..8718d4cab --- /dev/null +++ b/vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2020-28493.yml @@ -0,0 +1,26 @@ +--- +identifier: "CVE-2020-28493" +package_slug: "pypi/Jinja2" +title: "Regular Expression Denial of Service" +description: "The ReDOS vulnerability of the regex is mainly due to the sub-pattern + `[a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+` This issue can be mitigated by Markdown to format + user content instead of the urlize filter, or by implementing request timeouts and + limiting process memory." +date: "2021-07-21" +pubdate: "2021-02-01" +affected_range: "<2.11.3" +fixed_versions: +- "2.11.3" +affected_versions: "All versions before 2.11.3" +not_impacted: "All versions starting from 2.11.3" +solution: "Upgrade to version 2.11.3 or above." +urls: +- "https://nvd.nist.gov/vuln/detail/CVE-2020-28493" +cvss_v2: "AV:N/AC:L/Au:N/C:N/I:N/A:P" +cvss_v3: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:L" +uuid: "17c4602a-c0ae-48d0-96f6-08c84d55d0cb" +cwe_ids: +- "CWE-1035" +- "CWE-937" +identifiers: +- "CVE-2020-28493" diff --git a/vulntotal/tests/test_gitlab.py b/vulntotal/tests/test_gitlab.py new file mode 100644 index 000000000..3ad7162d6 --- /dev/null +++ b/vulntotal/tests/test_gitlab.py @@ -0,0 +1,56 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import gitlab + + +class TestGitlab(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "gitlab") + + def test_generate_package_advisory_url(self): + file_purls = self.get_test_loc("purls.txt") + with open(file_purls) as f: + purls = f.readlines() + results = [gitlab.get_package_slug(PackageURL.from_string(purl)) for purl in purls] + expected_file = self.get_test_loc("package_advisory_url-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_html_advisory(self): + advisory_folder = ( + Path(__file__) + .resolve() + .parent.joinpath("test_data/gitlab/temp_vulntotal_gitlab_datasource") + ) + results = [ + adv.to_dict() + for adv in gitlab.parse_interesting_advisories(advisory_folder, "0.1.1", False) + ] + expected_file = self.get_test_loc("parsed_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From edf2d610a10cb185d8af17a5701f8ab4b528963a Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 6 Sep 2022 00:45:39 +0530 Subject: [PATCH 19/36] register available datasources Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/__init__.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/vulntotal/datasources/__init__.py b/vulntotal/datasources/__init__.py index a2b13a650..d39dd560a 100644 --- a/vulntotal/datasources/__init__.py +++ b/vulntotal/datasources/__init__.py @@ -21,7 +21,27 @@ # VulnTotal is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +import glob +import importlib +import inspect +from os.path import basename +from os.path import dirname +from os.path import isfile +from os.path import join -DATASOURCE_REGISTRY = [] +from vulntotal.validator import DataSource -DATASOURCE_REGISTRY = {x.__module__.split(".")[-1]: x for x in DATASOURCE_REGISTRY} +DATASOURCE_REGISTRY = {} +files = glob.glob(join(dirname(__file__), "*.py")) +modules = [ + f"vulntotal.datasources.{basename(f)[:-3]}" + for f in files + if isfile(f) and not f.endswith("__init__.py") +] + + +for module in modules: + for name, cls in inspect.getmembers(importlib.import_module(module), inspect.isclass): + if cls.__module__ == module and cls.__base__ == DataSource: + DATASOURCE_REGISTRY[cls.__module__.split(".")[-1]] = cls + break From 1fa44ce55516b9d9d294a8c68ae4ffec68b26d73 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 15 Sep 2022 22:54:56 +0530 Subject: [PATCH 20/36] add osv DataSource Signed-off-by: Keshav Priyadarshi --- vulntotal/__init__.py | 8 ++ vulntotal/datasources/osv.py | 140 +++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 vulntotal/__init__.py create mode 100644 vulntotal/datasources/osv.py diff --git a/vulntotal/__init__.py b/vulntotal/__init__.py new file mode 100644 index 000000000..bdac1cd30 --- /dev/null +++ b/vulntotal/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/vulntotal/datasources/osv.py b/vulntotal/datasources/osv.py new file mode 100644 index 000000000..97ca7d38e --- /dev/null +++ b/vulntotal/datasources/osv.py @@ -0,0 +1,140 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from typing import Iterable + +import requests +from packageurl import PackageURL + +from vulntotal.ecosystem.nuget import get_closest_nuget_package_name +from vulntotal.validator import DataSource +from vulntotal.validator import VendorData +from vulntotal.vulntotal_utils import get_item + +logger = logging.getLogger(__name__) + + +class OSVDataSource(DataSource): + spdx_license_expression = "Apache-2.0" + license_url = "https://github.com/google/osv/blob/master/LICENSE" + url = "https://api.osv.dev/v1/query" + + def fetch_advisory(self, payload): + """Fetch JSON advisory from OSV API for a given package payload """ + + response = requests.post(self.url, data=str(payload)) + if not response.status_code == 200: + logger.error(f"Error while fetching {payload}: {response.status_code}") + return + return response.json() + + def datasource_advisory(self, purl) -> Iterable[VendorData]: + payload = generate_payload(purl) + if not payload: + return + advisory = self.fetch_advisory(payload) + self._raw_dump.append(advisory) + return parse_advisory(advisory) + + @classmethod + def supported_ecosystem(cls): + # source https://ossf.github.io/osv-schema/ + return { + "npm": "npm", + "maven": "Maven", + "golang": "Go", + "nuget": "NuGet", + "pypi": "PyPI", + "rubygems": "RubyGems", + "crates.io": "crates.io", + "composer": "Packagist", + "linux": "Linux", + "oss-fuzz": "OSS-Fuzz", + "debian": "Debian", + "hex": "Hex", + "android": "Android", + } + + +def parse_advisory(response) -> Iterable[VendorData]: + """Parse response from OSV API and yield VendorData""" + + for vuln in response.get("vulns") or []: + aliases = [] + affected_versions = [] + fixed = [] + + aliases.extend(vuln.get("aliases") or []) + aliases.append(vuln.get("id")) if vuln.get("id") else None + + try: + affected_versions.extend(get_item(vuln, "affected", 0, "versions") or []) + except: + pass + + try: + for event in get_item(vuln, "affected", 0, "ranges", 0, "events") or []: + affected_versions.append(event.get("introduced")) if event.get( + "introduced" + ) else None + fixed.append(event.get("fixed")) if event.get("fixed") else None + except: + pass + + yield VendorData( + aliases=sorted(list(set(aliases))), + affected_versions=sorted(list(set(affected_versions))), + fixed_versions=sorted(list(set(fixed))), + ) + + +def generate_payload(purl): + """Generate compatible payload for OSV API from a PURL""" + + supported_ecosystem = OSVDataSource.supported_ecosystem() + payload = {} + payload["version"] = purl.version + payload["package"] = {} + + if purl.type in supported_ecosystem: + payload["package"]["ecosystem"] = supported_ecosystem[purl.type] + + if purl.type == "maven": + if not purl.namespace: + logger.error(f"Invalid Maven PURL {str(purl)}") + return + payload["package"]["name"] = f"{purl.namespace}:{purl.name}" + + elif purl.type == "packagist": + if not purl.namespace: + logger.error(f"Invalid Packagist PURL {str(purl)}") + return + payload["package"]["name"] = f"{purl.namespace}/{purl.name}" + + elif purl.type == "linux": + if purl.name not in ("kernel", "Kernel"): + logger.error(f"Invalid Linux PURL {str(purl)}") + return + payload["package"]["name"] = "Kernel" + + elif purl.type == "nuget": + nuget_package = get_closest_nuget_package_name(purl.name) + if not nuget_package: + logger.error(f"Invalid NuGet PURL {str(purl)}") + return + payload["package"]["name"] = nuget_package + + elif purl.type == "golang" and purl.namespace: + payload["package"]["name"] = f"{purl.namespace}/{purl.name}" + + else: + payload["package"]["name"] = purl.name + + return payload From ab161ff6a76c92ec815c0d01460ce1a925c3e985 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 15 Sep 2022 22:56:32 +0530 Subject: [PATCH 21/36] support get_closest_nuget_package_name Signed-off-by: Keshav Priyadarshi --- vulntotal/ecosystem/__init__.py | 8 ++++++++ vulntotal/ecosystem/nuget.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 vulntotal/ecosystem/__init__.py create mode 100644 vulntotal/ecosystem/nuget.py diff --git a/vulntotal/ecosystem/__init__.py b/vulntotal/ecosystem/__init__.py new file mode 100644 index 000000000..bdac1cd30 --- /dev/null +++ b/vulntotal/ecosystem/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/vulntotal/ecosystem/nuget.py b/vulntotal/ecosystem/nuget.py new file mode 100644 index 000000000..1b459adc7 --- /dev/null +++ b/vulntotal/ecosystem/nuget.py @@ -0,0 +1,33 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from urllib.parse import urljoin + +import requests + + +def get_closest_nuget_package_name(query): + """ + Return case-sensitive NuGet package name using + SearchQueryService provided by NuGet + """ + url_nuget_service = "https://api.nuget.org/v3/index.json" + url_nuget_search = "" + + api_resources = requests.get(url_nuget_service).json() + for resource in api_resources.get("resources") or []: + if resource.get("@type") == "SearchQueryService": + url_nuget_search = resource["@id"] + break + + if url_nuget_search: + url_query = urljoin(url_nuget_search, f"?q={query}") + query_response = requests.get(url_query).json() + if query_response.get("data"): + return query_response["data"][0]["id"] From 491c318ecc53712400e06c294c2d07e00ef73b72 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 15 Sep 2022 23:05:34 +0530 Subject: [PATCH 22/36] test OSVDataSource Signed-off-by: Keshav Priyadarshi --- vulntotal/tests/test_data/osv/advisory.txt | 1 + .../osv/parse_advisory_data-expected.json | 195 ++++++++++++++++++ .../test_data/osv/payloads_data-expected.json | 70 +++++++ vulntotal/tests/test_data/osv/purls.txt | 10 + vulntotal/tests/test_osv.py | 37 ++++ 5 files changed, 313 insertions(+) create mode 100644 vulntotal/tests/test_data/osv/advisory.txt create mode 100644 vulntotal/tests/test_data/osv/parse_advisory_data-expected.json create mode 100644 vulntotal/tests/test_data/osv/payloads_data-expected.json create mode 100644 vulntotal/tests/test_data/osv/purls.txt create mode 100644 vulntotal/tests/test_osv.py diff --git a/vulntotal/tests/test_data/osv/advisory.txt b/vulntotal/tests/test_data/osv/advisory.txt new file mode 100644 index 000000000..e4e42849b --- /dev/null +++ b/vulntotal/tests/test_data/osv/advisory.txt @@ -0,0 +1 @@ +{"vulns":[{"id":"PYSEC-2014-8","details":"The default configuration for bccache.FileSystemBytecodeCache in Jinja2 before 2.7.2 does not properly create temporary files, which allows local users to gain privileges via a crafted .cache file with a name starting with __jinja2_ in /tmp.","aliases":["CVE-2014-1402"],"modified":"2021-07-05T00:01:22.043149Z","published":"2014-05-19T14:55:00Z","references":[{"type":"WEB","url":"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747"},{"type":"ADVISORY","url":"http://advisories.mageia.org/MGASA-2014-0028.html"},{"type":"WEB","url":"http://jinja.pocoo.org/docs/changelog/"},{"type":"WEB","url":"http://openwall.com/lists/oss-security/2014/01/10/3"},{"type":"REPORT","url":"https://bugzilla.redhat.com/show_bug.cgi?id=1051421"},{"type":"ADVISORY","url":"http://www.mandriva.com/security/advisories?name=MDVSA-2014:096"},{"type":"WEB","url":"http://openwall.com/lists/oss-security/2014/01/10/2"},{"type":"ADVISORY","url":"http://secunia.com/advisories/59017"},{"type":"ADVISORY","url":"http://secunia.com/advisories/58918"},{"type":"ADVISORY","url":"http://secunia.com/advisories/60770"},{"type":"ADVISORY","url":"http://secunia.com/advisories/60738"},{"type":"ADVISORY","url":"http://secunia.com/advisories/56287"},{"type":"WEB","url":"http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml"},{"type":"WEB","url":"https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html"},{"type":"ADVISORY","url":"http://secunia.com/advisories/58783"},{"type":"ADVISORY","url":"http://rhn.redhat.com/errata/RHSA-2014-0748.html"},{"type":"ADVISORY","url":"http://rhn.redhat.com/errata/RHSA-2014-0747.html"}],"affected":[{"package":{"name":"jinja2","ecosystem":"PyPI","purl":"pkg:pypi/jinja2"},"ranges":[{"type":"ECOSYSTEM","events":[{"introduced":"0"},{"fixed":"2.7.2"}]}],"versions":["2.0","2.0rc1","2.1","2.1.1","2.2","2.2.1","2.3","2.3.1","2.4","2.4.1","2.5","2.5.1","2.5.2","2.5.3","2.5.4","2.5.5","2.6","2.7","2.7.1"],"database_specific":{"source":"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-8.yaml"}}],"schema_version":"1.2.0"},{"id":"PYSEC-2014-82","details":"FileSystemBytecodeCache in Jinja2 2.7.2 does not properly create temporary directories, which allows local users to gain privileges by pre-creating a temporary directory with a user's uid. NOTE: this vulnerability exists because of an incomplete fix for CVE-2014-1402.","aliases":["CVE-2014-0012"],"modified":"2021-08-27T03:22:05.027573Z","published":"2014-05-19T14:55:00Z","references":[{"type":"REPORT","url":"https://bugzilla.redhat.com/show_bug.cgi?id=1051421"},{"type":"WEB","url":"https://github.com/mitsuhiko/jinja2/pull/292"},{"type":"FIX","url":"https://github.com/mitsuhiko/jinja2/commit/acb672b6a179567632e032f547582f30fa2f4aa7"},{"type":"WEB","url":"http://seclists.org/oss-sec/2014/q1/73"},{"type":"WEB","url":"https://github.com/mitsuhiko/jinja2/pull/296"},{"type":"ADVISORY","url":"http://secunia.com/advisories/60738"},{"type":"WEB","url":"http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml"},{"type":"ADVISORY","url":"http://secunia.com/advisories/56328"}],"affected":[{"package":{"name":"jinja2","ecosystem":"PyPI","purl":"pkg:pypi/jinja2"},"ranges":[{"type":"GIT","repo":"https://github.com/mitsuhiko/jinja2","events":[{"introduced":"0"},{"fixed":"acb672b6a179567632e032f547582f30fa2f4aa7"}]},{"type":"ECOSYSTEM","events":[{"introduced":"0"},{"fixed":"2.7.3"}]}],"versions":["2.0","2.0rc1","2.1","2.1.1","2.2","2.2.1","2.3","2.3.1","2.4","2.4.1","2.5","2.5.1","2.5.2","2.5.3","2.5.4","2.5.5","2.6","2.7","2.7.1","2.7.2"],"database_specific":{"source":"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-82.yaml"}}],"schema_version":"1.2.0"},{"id":"PYSEC-2019-217","details":"In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.","aliases":["CVE-2019-10906","GHSA-462w-v97r-4m45"],"modified":"2021-11-22T04:57:52.862665Z","published":"2019-04-07T00:29:00Z","references":[{"type":"ARTICLE","url":"https://palletsprojects.com/blog/jinja-2-10-1-released"},{"type":"WEB","url":"https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E"},{"type":"WEB","url":"https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E"},{"type":"WEB","url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/"},{"type":"WEB","url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/"},{"type":"WEB","url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1152"},{"type":"WEB","url":"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1237"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1329"},{"type":"WEB","url":"https://usn.ubuntu.com/4011-1/"},{"type":"WEB","url":"https://usn.ubuntu.com/4011-2/"},{"type":"WEB","url":"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html"},{"type":"ADVISORY","url":"https://github.com/advisories/GHSA-462w-v97r-4m45"}],"affected":[{"package":{"name":"jinja2","ecosystem":"PyPI","purl":"pkg:pypi/jinja2"},"ranges":[{"type":"ECOSYSTEM","events":[{"introduced":"0"},{"fixed":"2.10.1"}]}],"versions":["2.0","2.0rc1","2.1","2.1.1","2.10","2.2","2.2.1","2.3","2.3.1","2.4","2.4.1","2.5","2.5.1","2.5.2","2.5.3","2.5.4","2.5.5","2.6","2.7","2.7.1","2.7.2","2.7.3","2.8","2.8.1","2.9","2.9.1","2.9.2","2.9.3","2.9.4","2.9.5","2.9.6"],"database_specific":{"source":"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml"}}],"schema_version":"1.2.0"},{"id":"PYSEC-2019-220","details":"In Pallets Jinja before 2.8.1, str.format allows a sandbox escape.","aliases":["CVE-2016-10745","GHSA-hj2j-77xm-mc5v"],"modified":"2021-11-22T04:57:52.929678Z","published":"2019-04-08T13:29:00Z","references":[{"type":"ARTICLE","url":"https://palletsprojects.com/blog/jinja-281-released/"},{"type":"FIX","url":"https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1022"},{"type":"WEB","url":"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1237"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:1260"},{"type":"WEB","url":"https://usn.ubuntu.com/4011-1/"},{"type":"WEB","url":"https://usn.ubuntu.com/4011-2/"},{"type":"WEB","url":"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:3964"},{"type":"ADVISORY","url":"https://access.redhat.com/errata/RHSA-2019:4062"},{"type":"ADVISORY","url":"https://github.com/advisories/GHSA-hj2j-77xm-mc5v"}],"affected":[{"package":{"name":"jinja2","ecosystem":"PyPI","purl":"pkg:pypi/jinja2"},"ranges":[{"type":"GIT","repo":"https://github.com/pallets/jinja","events":[{"introduced":"0"},{"fixed":"9b53045c34e61013dc8f09b7e52a555fa16bed16"}]},{"type":"ECOSYSTEM","events":[{"introduced":"0"},{"fixed":"2.8.1"}]}],"versions":["2.0","2.0rc1","2.1","2.1.1","2.2","2.2.1","2.3","2.3.1","2.4","2.4.1","2.5","2.5.1","2.5.2","2.5.3","2.5.4","2.5.5","2.6","2.7","2.7.1","2.7.2","2.7.3","2.8"],"database_specific":{"source":"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-220.yaml"}}],"schema_version":"1.2.0"},{"id":"PYSEC-2021-66","details":"This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDoS vulnerability is mainly due to the `_punctuation_re regex` operator and its use of multiple wildcards. The last wildcard is the most exploitable as it searches for trailing punctuation. This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.","aliases":["CVE-2020-28493","SNYK-PYTHON-JINJA2-1012994","GHSA-g3rq-g295-4j3m"],"modified":"2021-03-22T16:34:00Z","published":"2021-02-01T20:15:00Z","references":[{"type":"WEB","url":"https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20"},{"type":"WEB","url":"https://github.com/pallets/jinja/pull/1343"},{"type":"ADVISORY","url":"https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994"},{"type":"WEB","url":"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/"},{"type":"ADVISORY","url":"https://github.com/advisories/GHSA-g3rq-g295-4j3m"}],"affected":[{"package":{"name":"jinja2","ecosystem":"PyPI","purl":"pkg:pypi/jinja2"},"ranges":[{"type":"ECOSYSTEM","events":[{"introduced":"0"},{"fixed":"2.11.3"}]}],"versions":["2.0rc1","2.0","2.1","2.1.1","2.2","2.2.1","2.3","2.3.1","2.4","2.4.1","2.5","2.5.1","2.5.2","2.5.3","2.5.4","2.5.5","2.6","2.7","2.7.1","2.7.2","2.7.3","2.8","2.8.1","2.9","2.9.1","2.9.2","2.9.3","2.9.4","2.9.5","2.9.6","2.10","2.10.1","2.10.2","2.10.3","2.11.0","2.11.1","2.11.2"],"database_specific":{"source":"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2021-66.yaml"}}],"schema_version":"1.2.0"}]} \ No newline at end of file diff --git a/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json b/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json new file mode 100644 index 000000000..c7bfc490c --- /dev/null +++ b/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json @@ -0,0 +1,195 @@ +[ + { + "affected_versions": [ + "0", + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1" + ], + "fixed_versions": [ + "2.7.2" + ], + "aliases": [ + "CVE-2014-1402", + "PYSEC-2014-8" + ] + }, + { + "affected_versions": [ + "0", + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1", + "2.7.2" + ], + "fixed_versions": [ + "acb672b6a179567632e032f547582f30fa2f4aa7" + ], + "aliases": [ + "CVE-2014-0012", + "PYSEC-2014-82" + ] + }, + { + "affected_versions": [ + "0", + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.10", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1", + "2.7.2", + "2.7.3", + "2.8", + "2.8.1", + "2.9", + "2.9.1", + "2.9.2", + "2.9.3", + "2.9.4", + "2.9.5", + "2.9.6" + ], + "fixed_versions": [ + "2.10.1" + ], + "aliases": [ + "CVE-2019-10906", + "GHSA-462w-v97r-4m45", + "PYSEC-2019-217" + ] + }, + { + "affected_versions": [ + "0", + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1", + "2.7.2", + "2.7.3", + "2.8" + ], + "fixed_versions": [ + "9b53045c34e61013dc8f09b7e52a555fa16bed16" + ], + "aliases": [ + "CVE-2016-10745", + "GHSA-hj2j-77xm-mc5v", + "PYSEC-2019-220" + ] + }, + { + "affected_versions": [ + "0", + "2.0", + "2.0rc1", + "2.1", + "2.1.1", + "2.10", + "2.10.1", + "2.10.2", + "2.10.3", + "2.11.0", + "2.11.1", + "2.11.2", + "2.2", + "2.2.1", + "2.3", + "2.3.1", + "2.4", + "2.4.1", + "2.5", + "2.5.1", + "2.5.2", + "2.5.3", + "2.5.4", + "2.5.5", + "2.6", + "2.7", + "2.7.1", + "2.7.2", + "2.7.3", + "2.8", + "2.8.1", + "2.9", + "2.9.1", + "2.9.2", + "2.9.3", + "2.9.4", + "2.9.5", + "2.9.6" + ], + "fixed_versions": [ + "2.11.3" + ], + "aliases": [ + "CVE-2020-28493", + "GHSA-g3rq-g295-4j3m", + "PYSEC-2021-66", + "SNYK-PYTHON-JINJA2-1012994" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/osv/payloads_data-expected.json b/vulntotal/tests/test_data/osv/payloads_data-expected.json new file mode 100644 index 000000000..2710217c3 --- /dev/null +++ b/vulntotal/tests/test_data/osv/payloads_data-expected.json @@ -0,0 +1,70 @@ +[ + { + "version": "2.4.1", + "package": { + "ecosystem": "PyPI", + "name": "jinja2" + } + }, + { + "version": "10", + "package": { + "ecosystem": "Android", + "name": "System" + } + }, + { + "version": "1.1.3-1", + "package": { + "name": "davical" + } + }, + { + "version": "10.1.0-M8", + "package": { + "ecosystem": "Maven", + "name": "org.apache.tomcat:tomcat" + } + }, + { + "version": "v5.4.195", + "package": { + "ecosystem": "Linux", + "name": "Kernel" + } + }, + { + "version": "12.0.5", + "package": { + "name": "dolibarr/dolibarr" + } + }, + { + "version": "0.9.7", + "package": { + "ecosystem": "crates.io", + "name": "sha2" + } + }, + { + "version": "2.18.0", + "package": { + "ecosystem": "NuGet", + "name": "Moment.js" + } + }, + { + "version": "3.1.3", + "package": { + "ecosystem": "npm", + "name": "semver-regex" + } + }, + { + "version": "1.1.0", + "package": { + "ecosystem": "Go", + "name": "github.com/cloudflare/cfrpki" + } + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/osv/purls.txt b/vulntotal/tests/test_data/osv/purls.txt new file mode 100644 index 000000000..d911f1539 --- /dev/null +++ b/vulntotal/tests/test_data/osv/purls.txt @@ -0,0 +1,10 @@ +pkg:pypi/jinja2@2.4.1 +pkg:android/System@10 +pkg:debian:8/davical@1.1.3-1 +pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8 +pkg:linux/Kernel@v5.4.195 +pkg:packagist/dolibarr/dolibarr@12.0.5 +pkg:crates.io/sha2@0.9.7 +pkg:nuget/Moment.js@2.18.0 +pkg:npm/semver-regex@3.1.3 +pkg:golang/github.com/cloudflare/cfrpki@1.1.0 \ No newline at end of file diff --git a/vulntotal/tests/test_osv.py b/vulntotal/tests/test_osv.py new file mode 100644 index 000000000..96c8bd24e --- /dev/null +++ b/vulntotal/tests/test_osv.py @@ -0,0 +1,37 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import osv + + +class TestOSV(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "osv") + + def test_generate_payload(self): + file_purls = self.get_test_loc("purls.txt") + with open(file_purls) as f: + purls = f.readlines() + results = [osv.generate_payload(PackageURL.from_string(purl)) for purl in purls] + expected_file = self.get_test_loc("payloads_data-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_advisory(self): + advisory_page = self.get_test_loc("advisory.txt") + with open(advisory_page) as f: + advisory = json.load(f) + results = [adv.to_dict() for adv in osv.parse_advisory(advisory)] + expected_file = self.get_test_loc("parse_advisory_data-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From 536dd914236d71cb0294ca5cac9b7e7ad55c30ff Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 15 Sep 2022 23:06:55 +0530 Subject: [PATCH 23/36] add utilities for vulntotal Signed-off-by: Keshav Priyadarshi --- vulntotal/vulntotal_utils.py | 190 +++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 vulntotal/vulntotal_utils.py diff --git a/vulntotal/vulntotal_utils.py b/vulntotal/vulntotal_utils.py new file mode 100644 index 000000000..bb422c99b --- /dev/null +++ b/vulntotal/vulntotal_utils.py @@ -0,0 +1,190 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import operator +from typing import Union + + +class GenericVersion: + def __init__(self, version): + self.value = version.replace(" ", "").lstrip("v") + + self.decomposed = tuple( + [int(com) if com.isnumeric() else com for com in self.value.split(".")] + ) + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__eq__(other.value) + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + for i, j in zip(self.decomposed, other.decomposed): + if not isinstance(i, type(j)): + continue + if i.__gt__(j): + return False + return True + + def __le__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.__lt__(other) or self.__eq__(other) + + +def compare(version, package_comparator, package_version): + operator_comparator = { + "<": operator.lt, + ">": operator.gt, + "=": operator.eq, + "<=": operator.le, + ">=": operator.ge, + "==": operator.eq, + "!=": operator.ne, + ")": operator.lt, + "]": operator.le, + "(": operator.gt, + "[": operator.ge, + } + compare = operator_comparator[package_comparator] + return compare(version, package_version) + + +def parse_constraint(constraint): + """ + Return operator and version from a constraint + For example: + >>> assert parse_constraint(">=7.0.0") == ('>=', '7.0.0') + >>> assert parse_constraint("=7.0.0") == ('=', '7.0.0') + >>> assert parse_constraint("[3.0.0") == ('[', '3.0.0') + >>> assert parse_constraint("3.1.25]") == (']', '3.1.25') + """ + if constraint.startswith(("<=", ">=", "==", "!=")): + return constraint[:2], constraint[2:] + + if constraint.startswith(("<", ">", "=", "[", "(")): + return constraint[0], constraint[1:] + + if constraint.endswith(("]", ")")): + return constraint[-1], constraint[:-1] + + +def github_constraints_satisfied(github_constrain, version): + """ + Return True or False depending on whether the given version satisfies the github constrain + For example: + >>> assert github_constraints_satisfied(">= 7.0.0, <= 7.6.57", "7.1.1") == True + >>> assert github_constraints_satisfied(">= 10.4.0, <= 10.4.1", "10.6.0") == False + """ + gh_constraints = github_constrain.strip().replace(" ", "") + constraints = gh_constraints.split(",") + for constraint in constraints: + gh_comparator, gh_version = parse_constraint(constraint) + if not gh_version: + continue + if not compare(GenericVersion(version), gh_comparator, GenericVersion(gh_version)): + return False + return True + + +def snky_constraints_satisfied(snyk_constrain, version): + """ + Return True or False depending on whether the given version satisfies the snyk constrain + For example: + >>> assert snky_constraints_satisfied(">=4.0.0, <4.0.10.16", "4.0.10.15") == True + >>> assert snky_constraints_satisfied(" >=4.1.0, <4.4.15.7", "4.0.10.15") == False + >>> assert snky_constraints_satisfied("[3.0.0,3.1.25)", "3.0.2") == True + """ + snyk_constraints = snyk_constrain.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + for constraint in constraints: + snyk_comparator, snyk_version = parse_constraint(constraint) + if not snyk_version: + continue + if not compare(GenericVersion(version), snyk_comparator, GenericVersion(snyk_version)): + return False + return True + + +def gitlab_constraints_satisfied(gitlab_constrain, version): + """ + Return True or False depending on whether the given version satisfies the gitlab constrain + For example: + >>> assert gitlab_constraints_satisfied("[7.0.0,7.0.11),[7.2.0,7.2.4)", "7.2.1") == True + >>> assert gitlab_constraints_satisfied("[7.0.0,7.0.11),[7.2.0,7.2.4)", "8.2.1") == False + >>> assert gitlab_constraints_satisfied( ">=4.0,<4.3||>=5.0,<5.2", "5.4") == False + >>> assert gitlab_constraints_satisfied( ">=0.19.0 <0.30.0", "0.24") == True + >>> assert gitlab_constraints_satisfied( ">=1.5,<1.5.2", "2.2") == False + """ + + gitlab_constraints = gitlab_constrain.strip() + if gitlab_constraints.startswith(("[", "(")): + # transform "[7.0.0,7.0.11),[7.2.0,7.2.4)" -> [ "[7.0.0,7.0.11)", "[7.2.0,7.2.4)" ] + splitted = gitlab_constraints.split(",") + constraints = [f"{a},{b}" for a, b in zip(splitted[::2], splitted[1::2])] + delimiter = "," + + else: + # transform ">=4.0,<4.3||>=5.0,<5.2" -> [ ">=4.0,<4.3", ">=5.0,<5.2" ] + # transform ">=0.19.0 <0.30.0" -> [ ">=0.19.0 <0.30.0" ] + # transform ">=1.5,<1.5.2" -> [ ">=1.5,<1.5.2" ] + delimiter = "," if "," in gitlab_constraints else " " + constraints = gitlab_constraints.split("||") + + for constraint in constraints: + is_constraint_satisfied = True + for subcontraint in constraint.strip().split(delimiter): + gitlab_comparator, gitlab_version = parse_constraint(subcontraint.strip()) + if not gitlab_version: + continue + if not compare( + GenericVersion(version), gitlab_comparator, GenericVersion(gitlab_version) + ): + is_constraint_satisfied = False + break + + if is_constraint_satisfied: + return True + return False + + +def get_item(entity: Union[dict, list], *attributes): + """ + Return `item` by going through all the `attributes` present in the `dictionary/list` + + Do a DFS for the `item` in the `dictionary/list` by traversing the `attributes` + and return None if can not traverse through the `attributes` + For example: + >>> get_item({'a': {'b': {'c': 'd'}}}, 'a', 'b', 'e') + Traceback (most recent call last): + ... + KeyError: "Missing attribute e in {'c': 'd'}" + >>> assert get_item({'a': {'b': {'c': 'd'}}}, 'a', 'b', 'c') == 'd' + >>> assert get_item({'a': [{'b': {'c': 'd'}}]}, 'a', 0, 'b') == {'c': 'd'} + >>> assert get_item(['b', ['c', ['d']]], 1, 1, 0) == 'd' + """ + for attribute in attributes: + if not entity: + return + if not isinstance(entity, (dict, list)): + raise TypeError(f"Entity must be of type `dict` or `list` not {type(entity)}") + if isinstance(entity, dict) and attribute not in entity: + raise KeyError(f"Missing attribute {attribute} in {entity}") + if isinstance(entity, list) and not isinstance(attribute, int): + raise TypeError(f"List indices must be integers not {type(attribute)}") + if isinstance(entity, list) and len(entity) <= attribute: + raise IndexError(f"Index {attribute} out of range for {entity}") + + entity = entity[attribute] + return entity From ec64fc968063355c0571dbd49a6e9e43f64eeac2 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 21:06:45 +0530 Subject: [PATCH 24/36] test snyk Datasource Signed-off-by: Keshav Priyadarshi --- .../snyk/extract_html_json-expected.json | 218 ++++++++++++++++++ .../snyk/package_advisory_url-expected.json | 13 ++ .../snyk/parsed_advisory-expected.json | 56 +++++ vulntotal/tests/test_data/snyk/purls.txt | 11 + vulntotal/tests/test_snyk.py | 63 +++++ 5 files changed, 361 insertions(+) create mode 100644 vulntotal/tests/test_data/snyk/extract_html_json-expected.json create mode 100644 vulntotal/tests/test_data/snyk/package_advisory_url-expected.json create mode 100644 vulntotal/tests/test_data/snyk/parsed_advisory-expected.json create mode 100644 vulntotal/tests/test_data/snyk/purls.txt create mode 100644 vulntotal/tests/test_snyk.py diff --git a/vulntotal/tests/test_data/snyk/extract_html_json-expected.json b/vulntotal/tests/test_data/snyk/extract_html_json-expected.json new file mode 100644 index 000000000..7babbeeb5 --- /dev/null +++ b/vulntotal/tests/test_data/snyk/extract_html_json-expected.json @@ -0,0 +1,218 @@ +[ + { + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-2401203": [ + "<1.4.3" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1924465": [ + "<1.4.1" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915648": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915560": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915390": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKISYNCLIB-1915559": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1915544": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1915643": [ + "<1.4.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1583445": [ + "<1.3.0" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORPKI-2401204": [ + "<1.4.3" + ], + "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORPKI-1915649": [ + "=21.10.0-beta.1, <21.10.8", + ">=22.4.0-beta.1, <22.4.1" + ], + "SNYK-PHP-CENTREONCENTREON-2971034": [ + "<21.4.16", + ">=21.10.0-beta.1, <21.10.8", + ">=22.4.0-beta.1, <22.4.1" + ], + "SNYK-PHP-CENTREONCENTREON-1567260": [ + ">=19.10.0, <20.4.0-beta.1", + ">=19.4.0, <19.4.15" + ], + "SNYK-PHP-CENTREONCENTREON-1536559": [ + ">=21.4.0, <21.4.2", + ">=20.10.0, <20.10.8", + "<20.4.14" + ], + "SNYK-PHP-CENTREONCENTREON-1536560": [ + ">=21.4.0, <21.4.2", + ">=20.10.0, <20.10.8", + "<20.4.14" + ], + "SNYK-PHP-CENTREONCENTREON-1536561": [ + ">=21.4.0, <21.4.2", + ">=20.10.0, <20.10.8", + "<20.4.14" + ], + "SNYK-PHP-CENTREONCENTREON-1534849": [ + ">=19.0.0, <19.4.5", + ">=18.10.0, <18.10.8", + "<2.8.30" + ], + "SNYK-PHP-CENTREONCENTREON-1320017": [ + "<20.4.13" + ], + "SNYK-PHP-CENTREONCENTREON-1320018": [ + "<20.4.13" + ], + "SNYK-PHP-CENTREONCENTREON-1296846": [ + "<21.4.0" + ], + "SNYK-PHP-CENTREONCENTREON-1247370": [ + "<2.8.37", + ">=20.10, <20.10.7", + ">=20.4, <20.4.13", + ">=19.10, <19.10.23" + ], + "SNYK-PHP-CENTREONCENTREON-1075031": [ + "<2.8.37", + ">=21.4, <21.4.1", + ">=20.10, <20.10.7", + ">=20.4, <20.4.13", + ">=19.10, <19.10.23" + ], + "SNYK-PHP-CENTREONCENTREON-570529": [ + ">=0.0.0, <1.6.4", + ">=18.10.0, <18.10.5", + ">=19.4.0, <19.4.3", + ">=19.10.0-beta.1, <19.10.2" + ], + "SNYK-PHP-CENTREONCENTREON-570528": [ + ">=0.0.0, <1.6.4", + ">=18.10.0, <18.10.5", + ">=19.4.0, <19.4.3", + ">=19.10.0-beta.1, <19.10.2" + ], + "SNYK-PHP-CENTREONCENTREON-570527": [ + "<19.10.7" + ], + "SNYK-PHP-CENTREONCENTREON-570051": [ + "<19.4.15" + ], + "SNYK-PHP-CENTREONCENTREON-564443": [ + "<19.10.13" + ], + "SNYK-PHP-CENTREONCENTREON-560860": [ + "<19.4.5" + ], + "SNYK-PHP-CENTREONCENTREON-560859": [ + "<19.4.5" + ], + "SNYK-PHP-CENTREONCENTREON-560847": [ + "<19.4.5" + ], + "SNYK-PHP-CENTREONCENTREON-559334": [ + ">=18.10.6, <18.10.8", + ">=19.10.0, <19.10.2", + ">=19.04.2, <19.04.5" + ], + "SNYK-PHP-CENTREONCENTREON-559444": [ + ">=19.10.0, <19.10.2", + ">=19.4.0, <19.4.5", + ">=18.10.0, <18.10.8", + "<2.8.30" + ], + "SNYK-PHP-CENTREONCENTREON-559445": [ + ">=19.10.0, <19.10.2", + ">=19.4.0, <19.4.5", + "<18.10.8" + ], + "SNYK-PHP-CENTREONCENTREON-559335": [ + ">=18.10.6, <18.10.9", + ">=19.10.0, <19.10.3", + ">=19.04.2, <19.04.7" + ], + "SNYK-PHP-CENTREONCENTREON-551996": [ + ">=19.10.0, <19.10.2", + "<19.4.5" + ], + "SNYK-PHP-CENTREONCENTREON-536206": [ + ">=19.10.0, <19.10.2", + ">=19.4.0, <19.4.5", + ">=18.10.0, <18.10.8", + ">=2.8.0, <2.8.30" + ], + "SNYK-PHP-CENTREONCENTREON-536202": [ + ">=19.10.0, <19.10.2", + ">=18.10.0, <18.10.8", + ">=2.8.0, <2.8.30", + ">=19.4.0, <19.4.5" + ], + "SNYK-PHP-CENTREONCENTREON-535972": [ + ">=18.10.6, <18.10.8", + ">=19.4.2, <19.4.5", + ">=2.7.3, <2.8.30" + ], + "SNYK-PHP-CENTREONCENTREON-473006": [ + ">=0.0.0" + ], + "SNYK-PHP-CENTREONCENTREON-472423": [ + ">=2.8.0, <2.8.28", + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472430": [ + ">=20.10.0, <20.10.3", + ">=20.4.0, <20.4.9", + ">=19.10.0, <19.10.19" + ], + "SNYK-PHP-CENTREONCENTREON-472429": [ + ">=2.8.0, <2.8.28", + ">=18.10.0, <18.10.5" + ], + "SNYK-PHP-CENTREONCENTREON-472428": [ + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472425": [ + ">=2.8.0, <2.8.27", + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472424": [ + ">=2.8.0, <18.10.5" + ], + "SNYK-PHP-CENTREONCENTREON-472422": [ + ">=2.8.0, <2.8.27", + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472418": [ + ">=2.8.0, <2.8.28", + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472371": [ + ">=2.8.0, <2.8.28", + ">=18.10.0, <18.10.4" + ], + "SNYK-PHP-CENTREONCENTREON-472370": [ + "<19.4.17" + ], + "SNYK-PHP-CENTREONCENTREON-469159": [ + "<19.10.0-rc.1" + ], + "SNYK-PHP-CENTREONCENTREON-451340": [ + "<19.10.0" + ], + "SNYK-PHP-CENTREONCENTREON-450214": [ + ">=18.10.0, <18.10.5", + ">=2.8.0, <2.8.28" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/snyk/package_advisory_url-expected.json b/vulntotal/tests/test_data/snyk/package_advisory_url-expected.json new file mode 100644 index 000000000..a7e4212df --- /dev/null +++ b/vulntotal/tests/test_data/snyk/package_advisory_url-expected.json @@ -0,0 +1,13 @@ +[ + "https://security.snyk.io/package/pip/jinja2", + "https://security.snyk.io/package/maven/org.apache.tomcat%3Atomcat", + "https://security.snyk.io/package/npm/semver-regex", + "https://security.snyk.io/package/golang/github.com%2Fmattermost%2Fmattermost-server%2Fv6%2Fapi4", + "https://security.snyk.io/package/composer/bolt%2Fcore", + "https://security.snyk.io/package/linux/debain:11/trafficserver", + "https://security.snyk.io/package/nuget/moment.js", + "https://security.snyk.io/package/cocoapods/ffmpeg", + "https://security.snyk.io/package/hex/coherence", + "https://security.snyk.io/package/rubygems/log4j-jars", + "https://security.snyk.io/api/listing?search=firefox&type=unmanaged" +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/snyk/parsed_advisory-expected.json b/vulntotal/tests/test_data/snyk/parsed_advisory-expected.json new file mode 100644 index 000000000..5d5468f38 --- /dev/null +++ b/vulntotal/tests/test_data/snyk/parsed_advisory-expected.json @@ -0,0 +1,56 @@ +[ + { + "affected_versions": [ + "TEST-AFFECTED" + ], + "fixed_versions": [ + "6.0.3" + ], + "aliases": [ + "CVE-2021-37861", + "TEST-SNYKID" + ] + }, + { + "affected_versions": [ + "TEST-AFFECTED" + ], + "fixed_versions": [ + "0.6.1" + ], + "aliases": [ + "CVE-2021-37863", + "TEST-SNYKID" + ] + }, + { + "affected_versions": [ + "TEST-AFFECTED" + ], + "fixed_versions": [ + "5.37.9", + "6.2.5", + "6.3.5", + "6.4.2" + ], + "aliases": [ + "CVE-2022-1332", + "TEST-SNYKID" + ] + }, + { + "affected_versions": [ + "TEST-AFFECTED" + ], + "fixed_versions": [ + "5.37.7", + "6.1.2", + "6.2.2", + "6.3.1" + ], + "aliases": [ + "CVE-2022-0708", + "TEST-SNYKID" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/snyk/purls.txt b/vulntotal/tests/test_data/snyk/purls.txt new file mode 100644 index 000000000..80628ad1c --- /dev/null +++ b/vulntotal/tests/test_data/snyk/purls.txt @@ -0,0 +1,11 @@ +pkg:pypi/jinja2@2.4.1 +pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8 +pkg:npm/semver-regex@3.1.3 +pkg:golang/github.com/mattermost/mattermost-server/v6/api4@0.1 +pkg:composer/bolt/core@0.1 +pkg:linux/trafficserver@5.4.1?distro=debain:11 +pkg:nuget/moment.js@2.18.0 +pkg:cocoapods/ffmpeg@0.2 +pkg:hex/coherence@0.2.1 +pkg:rubygems/log4j-jars@0.2 +pkg:unmanaged/firefox@8.9.1 \ No newline at end of file diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py new file mode 100644 index 000000000..1cf8d5a23 --- /dev/null +++ b/vulntotal/tests/test_snyk.py @@ -0,0 +1,63 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import snyk + + +class TestSnyk(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "snyk") + + def test_generate_package_advisory_url(self): + file_purls = self.get_test_loc("purls.txt") + with open(file_purls) as f: + purls = f.readlines() + results = [ + snyk.generate_package_advisory_url(PackageURL.from_string(purl)) for purl in purls + ] + expected_file = self.get_test_loc("package_advisory_url-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_extract_html_json_advisories(self): + file = self.get_test_loc("raw_pacakage_advisories.json") + with open(file) as f: + pages = json.load(f) + results = [snyk.extract_html_json_advisories(i) for i in pages] + expected_file = self.get_test_loc("extract_html_json-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_parse_html_advisory(self): + file = self.get_test_loc("raw_html_advisory.json") + with open(file) as f: + pages = json.load(f) + results = [ + snyk.parse_html_advisory(i, "TEST-SNYKID", ["TEST-AFFECTED"]).to_dict() for i in pages + ] + expected_file = self.get_test_loc("parsed_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From fb7ef75442f170a29f7443b3d6de6010dfb9eda5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 5 Sep 2022 20:52:31 +0530 Subject: [PATCH 25/36] test VulnerableCodeDataSource Signed-off-by: Keshav Priyadarshi --- .../test_data/vulnerablecode/advisory.json | 1614 +++++++++++++++++ .../parse_advisory-expected.json | 400 ++++ vulntotal/tests/test_vulnerablecode.py | 43 + 3 files changed, 2057 insertions(+) create mode 100644 vulntotal/tests/test_data/vulnerablecode/advisory.json create mode 100644 vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json create mode 100644 vulntotal/tests/test_vulnerablecode.py diff --git a/vulntotal/tests/test_data/vulnerablecode/advisory.json b/vulntotal/tests/test_data/vulnerablecode/advisory.json new file mode 100644 index 000000000..56130fe79 --- /dev/null +++ b/vulntotal/tests/test_data/vulnerablecode/advisory.json @@ -0,0 +1,1614 @@ +[ + { + "url":"http://localhost:8001/api/vulnerabilities/7870", + "vulnerability_id":"VULCOID-62M", + "summary":"Cross-site Scripting in Apache Tomcat", + "aliases":[ + { + "alias":"GHSA-6j88-6whg-x687" + }, + { + "alias":"CVE-2022-34305" + } + ], + "fixed_packages":[ + { + "url":"http://localhost:8001/api/packages/100195", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.82" + }, + { + "url":"http://localhost:8001/api/packages/100221", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.65" + }, + { + "url":"http://localhost:8001/api/packages/100240", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.22" + }, + { + "url":"http://localhost:8001/api/packages/100255", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M17" + }, + { + "url":"http://localhost:8001/api/packages/72871", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M1" + } + ], + "affected_packages":[ + { + "url":"http://localhost:8001/api/packages/100194", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.50" + }, + { + "url":"http://localhost:8001/api/packages/100220", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.30" + }, + { + "url":"http://localhost:8001/api/packages/74553", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M1" + }, + { + "url":"http://localhost:8001/api/packages/100253", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M1" + }, + { + "url":"http://localhost:8001/api/packages/100254", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M16" + }, + { + "url":"http://localhost:8001/api/packages/108992", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.51" + }, + { + "url":"http://localhost:8001/api/packages/137900", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.53" + }, + { + "url":"http://localhost:8001/api/packages/137901", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.54" + }, + { + "url":"http://localhost:8001/api/packages/107840", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.55" + }, + { + "url":"http://localhost:8001/api/packages/107822", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.56" + }, + { + "url":"http://localhost:8001/api/packages/107835", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.57" + }, + { + "url":"http://localhost:8001/api/packages/137902", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.58" + }, + { + "url":"http://localhost:8001/api/packages/107695", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.59" + }, + { + "url":"http://localhost:8001/api/packages/108023", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.60" + }, + { + "url":"http://localhost:8001/api/packages/137903", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.61" + }, + { + "url":"http://localhost:8001/api/packages/137904", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.63" + }, + { + "url":"http://localhost:8001/api/packages/108090", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.64" + }, + { + "url":"http://localhost:8001/api/packages/108155", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.65" + }, + { + "url":"http://localhost:8001/api/packages/137905", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.66" + }, + { + "url":"http://localhost:8001/api/packages/107712", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.68" + }, + { + "url":"http://localhost:8001/api/packages/137906", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.69" + }, + { + "url":"http://localhost:8001/api/packages/137907", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.70" + }, + { + "url":"http://localhost:8001/api/packages/137908", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.71" + }, + { + "url":"http://localhost:8001/api/packages/108024", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.72" + }, + { + "url":"http://localhost:8001/api/packages/137909", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.73" + }, + { + "url":"http://localhost:8001/api/packages/107591", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.75" + }, + { + "url":"http://localhost:8001/api/packages/137910", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.76" + }, + { + "url":"http://localhost:8001/api/packages/137911", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.77" + }, + { + "url":"http://localhost:8001/api/packages/137912", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.78" + }, + { + "url":"http://localhost:8001/api/packages/137913", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.79" + }, + { + "url":"http://localhost:8001/api/packages/137914", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.81" + }, + { + "url":"http://localhost:8001/api/packages/109040", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.31" + }, + { + "url":"http://localhost:8001/api/packages/129719", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.33" + }, + { + "url":"http://localhost:8001/api/packages/129720", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.34" + }, + { + "url":"http://localhost:8001/api/packages/72872", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.35" + }, + { + "url":"http://localhost:8001/api/packages/107824", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.36" + }, + { + "url":"http://localhost:8001/api/packages/107836", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.37" + }, + { + "url":"http://localhost:8001/api/packages/137915", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.38" + }, + { + "url":"http://localhost:8001/api/packages/107697", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.39" + }, + { + "url":"http://localhost:8001/api/packages/108025", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.40" + }, + { + "url":"http://localhost:8001/api/packages/137916", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.41" + }, + { + "url":"http://localhost:8001/api/packages/137917", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.43" + }, + { + "url":"http://localhost:8001/api/packages/108092", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.44" + }, + { + "url":"http://localhost:8001/api/packages/108156", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.45" + }, + { + "url":"http://localhost:8001/api/packages/137918", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.46" + }, + { + "url":"http://localhost:8001/api/packages/107713", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.48" + }, + { + "url":"http://localhost:8001/api/packages/137919", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.50" + }, + { + "url":"http://localhost:8001/api/packages/137920", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.52" + }, + { + "url":"http://localhost:8001/api/packages/137921", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.53" + }, + { + "url":"http://localhost:8001/api/packages/108026", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.54" + }, + { + "url":"http://localhost:8001/api/packages/137922", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.55" + }, + { + "url":"http://localhost:8001/api/packages/137923", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.56" + }, + { + "url":"http://localhost:8001/api/packages/107893", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.58" + }, + { + "url":"http://localhost:8001/api/packages/137924", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.59" + }, + { + "url":"http://localhost:8001/api/packages/137925", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.60" + }, + { + "url":"http://localhost:8001/api/packages/137926", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.62" + }, + { + "url":"http://localhost:8001/api/packages/137927", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.63" + }, + { + "url":"http://localhost:8001/api/packages/137928", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.64" + }, + { + "url":"http://localhost:8001/api/packages/129721", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M3" + }, + { + "url":"http://localhost:8001/api/packages/74554", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + }, + { + "url":"http://localhost:8001/api/packages/74555", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M5" + }, + { + "url":"http://localhost:8001/api/packages/107825", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M6" + }, + { + "url":"http://localhost:8001/api/packages/107837", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M7" + }, + { + "url":"http://localhost:8001/api/packages/107698", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M8" + }, + { + "url":"http://localhost:8001/api/packages/107699", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M9" + }, + { + "url":"http://localhost:8001/api/packages/137929", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M10" + }, + { + "url":"http://localhost:8001/api/packages/107894", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0" + }, + { + "url":"http://localhost:8001/api/packages/137930", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.2" + }, + { + "url":"http://localhost:8001/api/packages/108093", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.4" + }, + { + "url":"http://localhost:8001/api/packages/108158", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.5" + }, + { + "url":"http://localhost:8001/api/packages/137931", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.6" + }, + { + "url":"http://localhost:8001/api/packages/107714", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.7" + }, + { + "url":"http://localhost:8001/api/packages/137932", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.8" + }, + { + "url":"http://localhost:8001/api/packages/137933", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.10" + }, + { + "url":"http://localhost:8001/api/packages/137934", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.11" + }, + { + "url":"http://localhost:8001/api/packages/108027", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.12" + }, + { + "url":"http://localhost:8001/api/packages/137935", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.13" + }, + { + "url":"http://localhost:8001/api/packages/137936", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.14" + }, + { + "url":"http://localhost:8001/api/packages/107895", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.16" + }, + { + "url":"http://localhost:8001/api/packages/137937", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.17" + }, + { + "url":"http://localhost:8001/api/packages/137938", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.18" + }, + { + "url":"http://localhost:8001/api/packages/137939", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.20" + }, + { + "url":"http://localhost:8001/api/packages/137940", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.21" + }, + { + "url":"http://localhost:8001/api/packages/137941", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M2" + }, + { + "url":"http://localhost:8001/api/packages/137942", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M4" + }, + { + "url":"http://localhost:8001/api/packages/108029", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M5" + }, + { + "url":"http://localhost:8001/api/packages/108030", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M6" + }, + { + "url":"http://localhost:8001/api/packages/137943", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M7" + }, + { + "url":"http://localhost:8001/api/packages/137944", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8" + }, + { + "url":"http://localhost:8001/api/packages/137945", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M10" + }, + { + "url":"http://localhost:8001/api/packages/137946", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M11" + }, + { + "url":"http://localhost:8001/api/packages/137947", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M12" + }, + { + "url":"http://localhost:8001/api/packages/137948", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M14" + }, + { + "url":"http://localhost:8001/api/packages/137949", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M15" + } + ], + "references":[ + { + "reference_url":"https://nvd.nist.gov/vuln/detail/CVE-2022-34305", + "reference_id":"CVE-2022-34305", + "scores":[ + { + "value":"6.1", + "scoring_system":"cvssv3" + }, + { + "value":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N", + "scoring_system":"cvssv3_vector" + }, + { + "value":"4.3", + "scoring_system":"cvssv2" + }, + { + "value":"AV:N/AC:M/Au:N/C:N/I:P/A:N", + "scoring_system":"cvssv2_vector" + } + ], + "url":"https://nvd.nist.gov/vuln/detail/CVE-2022-34305" + }, + { + "reference_url":"https://lists.apache.org/thread/k04zk0nq6w57m72w5gb0r6z9ryhmvr4k", + "reference_id":"", + "scores":[ + + ], + "url":"https://lists.apache.org/thread/k04zk0nq6w57m72w5gb0r6z9ryhmvr4k" + }, + { + "reference_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2022-34305.json", + "reference_id":"", + "scores":[ + { + "value":"5.4", + "scoring_system":"cvssv3" + }, + { + "value":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N", + "scoring_system":"cvssv3_vector" + } + ], + "url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2022-34305.json" + } + ] + }, + { + "url":"http://localhost:8001/api/vulnerabilities/8420", + "vulnerability_id":"VULCOID-6HW", + "summary":"Unrestricted Upload of File with Dangerous Type Apache Tomcat", + "aliases":[ + { + "alias":"GHSA-xjgh-84hx-56c5" + }, + { + "alias":"CVE-2017-12617" + } + ], + "fixed_packages":[ + { + "url":"http://localhost:8001/api/packages/107247", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.82" + }, + { + "url":"http://localhost:8001/api/packages/107249", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.47" + }, + { + "url":"http://localhost:8001/api/packages/107251", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.23" + }, + { + "url":"http://localhost:8001/api/packages/107253", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.1" + } + ], + "affected_packages":[ + { + "url":"http://localhost:8001/api/packages/93717", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.0" + }, + { + "url":"http://localhost:8001/api/packages/107246", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.81" + }, + { + "url":"http://localhost:8001/api/packages/100012", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.0" + }, + { + "url":"http://localhost:8001/api/packages/107248", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.46" + }, + { + "url":"http://localhost:8001/api/packages/107127", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.0" + }, + { + "url":"http://localhost:8001/api/packages/107250", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.22" + }, + { + "url":"http://localhost:8001/api/packages/107252", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0M27" + }, + { + "url":"http://localhost:8001/api/packages/72871", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M1" + }, + { + "url":"http://localhost:8001/api/packages/131037", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.35" + }, + { + "url":"http://localhost:8001/api/packages/131038", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.37" + }, + { + "url":"http://localhost:8001/api/packages/131039", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.39" + }, + { + "url":"http://localhost:8001/api/packages/106872", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.40" + }, + { + "url":"http://localhost:8001/api/packages/107312", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.41" + }, + { + "url":"http://localhost:8001/api/packages/131040", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.42" + }, + { + "url":"http://localhost:8001/api/packages/131041", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.47" + }, + { + "url":"http://localhost:8001/api/packages/106927", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.50" + }, + { + "url":"http://localhost:8001/api/packages/131042", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.52" + }, + { + "url":"http://localhost:8001/api/packages/106958", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.53" + }, + { + "url":"http://localhost:8001/api/packages/106962", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.54" + }, + { + "url":"http://localhost:8001/api/packages/106968", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.55" + }, + { + "url":"http://localhost:8001/api/packages/131043", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.56" + }, + { + "url":"http://localhost:8001/api/packages/131044", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.57" + }, + { + "url":"http://localhost:8001/api/packages/131045", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.59" + }, + { + "url":"http://localhost:8001/api/packages/131046", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.61" + }, + { + "url":"http://localhost:8001/api/packages/131047", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.62" + }, + { + "url":"http://localhost:8001/api/packages/131048", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.63" + }, + { + "url":"http://localhost:8001/api/packages/94062", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.64" + }, + { + "url":"http://localhost:8001/api/packages/94063", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.65" + }, + { + "url":"http://localhost:8001/api/packages/137848", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.67" + }, + { + "url":"http://localhost:8001/api/packages/107105", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.68" + }, + { + "url":"http://localhost:8001/api/packages/137849", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.69" + }, + { + "url":"http://localhost:8001/api/packages/107120", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.70" + }, + { + "url":"http://localhost:8001/api/packages/100014", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.72" + }, + { + "url":"http://localhost:8001/api/packages/107192", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.73" + }, + { + "url":"http://localhost:8001/api/packages/107294", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.75" + }, + { + "url":"http://localhost:8001/api/packages/107284", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.76" + }, + { + "url":"http://localhost:8001/api/packages/107299", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.77" + }, + { + "url":"http://localhost:8001/api/packages/107300", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.78" + }, + { + "url":"http://localhost:8001/api/packages/107313", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.79" + }, + { + "url":"http://localhost:8001/api/packages/131049", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.1" + }, + { + "url":"http://localhost:8001/api/packages/131050", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.3" + }, + { + "url":"http://localhost:8001/api/packages/131051", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.5" + }, + { + "url":"http://localhost:8001/api/packages/131052", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.8" + }, + { + "url":"http://localhost:8001/api/packages/106967", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.9" + }, + { + "url":"http://localhost:8001/api/packages/131053", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.11" + }, + { + "url":"http://localhost:8001/api/packages/131054", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.12" + }, + { + "url":"http://localhost:8001/api/packages/131055", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.14" + }, + { + "url":"http://localhost:8001/api/packages/131056", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.15" + }, + { + "url":"http://localhost:8001/api/packages/131057", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.17" + }, + { + "url":"http://localhost:8001/api/packages/131058", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.18" + }, + { + "url":"http://localhost:8001/api/packages/131059", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.20" + }, + { + "url":"http://localhost:8001/api/packages/131060", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.21" + }, + { + "url":"http://localhost:8001/api/packages/131061", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.22" + }, + { + "url":"http://localhost:8001/api/packages/131062", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.23" + }, + { + "url":"http://localhost:8001/api/packages/131063", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.24" + }, + { + "url":"http://localhost:8001/api/packages/94077", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.26" + }, + { + "url":"http://localhost:8001/api/packages/94078", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.27" + }, + { + "url":"http://localhost:8001/api/packages/137844", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.28" + }, + { + "url":"http://localhost:8001/api/packages/137845", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.29" + }, + { + "url":"http://localhost:8001/api/packages/107106", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.30" + }, + { + "url":"http://localhost:8001/api/packages/107121", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.32" + }, + { + "url":"http://localhost:8001/api/packages/137846", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.33" + }, + { + "url":"http://localhost:8001/api/packages/137847", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.35" + }, + { + "url":"http://localhost:8001/api/packages/107126", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.36" + }, + { + "url":"http://localhost:8001/api/packages/100013", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.37" + }, + { + "url":"http://localhost:8001/api/packages/141874", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.38" + }, + { + "url":"http://localhost:8001/api/packages/107194", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.39" + }, + { + "url":"http://localhost:8001/api/packages/107286", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.41" + }, + { + "url":"http://localhost:8001/api/packages/107287", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.42" + }, + { + "url":"http://localhost:8001/api/packages/107301", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.43" + }, + { + "url":"http://localhost:8001/api/packages/107302", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.44" + }, + { + "url":"http://localhost:8001/api/packages/107314", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.45" + }, + { + "url":"http://localhost:8001/api/packages/137842", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.2" + }, + { + "url":"http://localhost:8001/api/packages/137843", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.3" + }, + { + "url":"http://localhost:8001/api/packages/107128", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.4" + }, + { + "url":"http://localhost:8001/api/packages/100005", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.5" + }, + { + "url":"http://localhost:8001/api/packages/141875", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.6" + }, + { + "url":"http://localhost:8001/api/packages/107187", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.8" + }, + { + "url":"http://localhost:8001/api/packages/142151", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.9" + }, + { + "url":"http://localhost:8001/api/packages/142152", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.11" + }, + { + "url":"http://localhost:8001/api/packages/107288", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.12" + }, + { + "url":"http://localhost:8001/api/packages/107289", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.13" + }, + { + "url":"http://localhost:8001/api/packages/107303", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.14" + }, + { + "url":"http://localhost:8001/api/packages/107304", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.15" + }, + { + "url":"http://localhost:8001/api/packages/107315", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.16" + }, + { + "url":"http://localhost:8001/api/packages/142153", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.19" + }, + { + "url":"http://localhost:8001/api/packages/142154", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.20" + }, + { + "url":"http://localhost:8001/api/packages/142155", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.21" + }, + { + "url":"http://localhost:8001/api/packages/142156", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.84" + }, + { + "url":"http://localhost:8001/api/packages/142157", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.85" + }, + { + "url":"http://localhost:8001/api/packages/142158", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.86" + }, + { + "url":"http://localhost:8001/api/packages/142159", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.88" + }, + { + "url":"http://localhost:8001/api/packages/142160", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.90" + }, + { + "url":"http://localhost:8001/api/packages/142161", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.91" + }, + { + "url":"http://localhost:8001/api/packages/142162", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.92" + }, + { + "url":"http://localhost:8001/api/packages/142163", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.93" + }, + { + "url":"http://localhost:8001/api/packages/142164", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.94" + }, + { + "url":"http://localhost:8001/api/packages/142165", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.96" + }, + { + "url":"http://localhost:8001/api/packages/142166", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.99" + }, + { + "url":"http://localhost:8001/api/packages/109042", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.100" + }, + { + "url":"http://localhost:8001/api/packages/142167", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.103" + }, + { + "url":"http://localhost:8001/api/packages/107839", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.104" + }, + { + "url":"http://localhost:8001/api/packages/142168", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.105" + }, + { + "url":"http://localhost:8001/api/packages/142169", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.106" + }, + { + "url":"http://localhost:8001/api/packages/142170", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.107" + }, + { + "url":"http://localhost:8001/api/packages/142171", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.108" + }, + { + "url":"http://localhost:8001/api/packages/142172", + "purl":"pkg:maven/org.apache.tomcat/tomcat@7.0.109" + }, + { + "url":"http://localhost:8001/api/packages/106924", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.0-RC1" + }, + { + "url":"http://localhost:8001/api/packages/141872", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.0-RC3" + }, + { + "url":"http://localhost:8001/api/packages/141873", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.0-RC5" + }, + { + "url":"http://localhost:8001/api/packages/106926", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.0-RC10" + }, + { + "url":"http://localhost:8001/api/packages/142173", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.48" + }, + { + "url":"http://localhost:8001/api/packages/142174", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.49" + }, + { + "url":"http://localhost:8001/api/packages/142175", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.50" + }, + { + "url":"http://localhost:8001/api/packages/142176", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.51" + }, + { + "url":"http://localhost:8001/api/packages/142177", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.52" + }, + { + "url":"http://localhost:8001/api/packages/107834", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.0.53" + }, + { + "url":"http://localhost:8001/api/packages/142178", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.24" + }, + { + "url":"http://localhost:8001/api/packages/142179", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.27" + }, + { + "url":"http://localhost:8001/api/packages/142180", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.28" + }, + { + "url":"http://localhost:8001/api/packages/142181", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.29" + }, + { + "url":"http://localhost:8001/api/packages/142182", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.30" + }, + { + "url":"http://localhost:8001/api/packages/142183", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.31" + }, + { + "url":"http://localhost:8001/api/packages/142184", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.32" + }, + { + "url":"http://localhost:8001/api/packages/142185", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.33" + }, + { + "url":"http://localhost:8001/api/packages/142186", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.34" + }, + { + "url":"http://localhost:8001/api/packages/142187", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.35" + }, + { + "url":"http://localhost:8001/api/packages/142188", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.37" + }, + { + "url":"http://localhost:8001/api/packages/142189", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.38" + }, + { + "url":"http://localhost:8001/api/packages/142190", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.39" + }, + { + "url":"http://localhost:8001/api/packages/142191", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.40" + }, + { + "url":"http://localhost:8001/api/packages/142192", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.41" + }, + { + "url":"http://localhost:8001/api/packages/142193", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.42" + }, + { + "url":"http://localhost:8001/api/packages/142194", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.43" + }, + { + "url":"http://localhost:8001/api/packages/142195", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.45" + }, + { + "url":"http://localhost:8001/api/packages/142196", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.46" + }, + { + "url":"http://localhost:8001/api/packages/142197", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.47" + }, + { + "url":"http://localhost:8001/api/packages/142198", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.49" + }, + { + "url":"http://localhost:8001/api/packages/100194", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.50" + }, + { + "url":"http://localhost:8001/api/packages/108992", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.51" + }, + { + "url":"http://localhost:8001/api/packages/137900", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.53" + }, + { + "url":"http://localhost:8001/api/packages/137901", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.54" + }, + { + "url":"http://localhost:8001/api/packages/107840", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.55" + }, + { + "url":"http://localhost:8001/api/packages/107822", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.56" + }, + { + "url":"http://localhost:8001/api/packages/107835", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.57" + }, + { + "url":"http://localhost:8001/api/packages/137902", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.58" + }, + { + "url":"http://localhost:8001/api/packages/107695", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.59" + }, + { + "url":"http://localhost:8001/api/packages/108023", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.60" + }, + { + "url":"http://localhost:8001/api/packages/137903", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.61" + }, + { + "url":"http://localhost:8001/api/packages/137904", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.63" + }, + { + "url":"http://localhost:8001/api/packages/108090", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.64" + }, + { + "url":"http://localhost:8001/api/packages/108155", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.65" + }, + { + "url":"http://localhost:8001/api/packages/137905", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.66" + }, + { + "url":"http://localhost:8001/api/packages/107712", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.68" + }, + { + "url":"http://localhost:8001/api/packages/137906", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.69" + }, + { + "url":"http://localhost:8001/api/packages/137907", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.70" + }, + { + "url":"http://localhost:8001/api/packages/137908", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.71" + }, + { + "url":"http://localhost:8001/api/packages/108024", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.72" + }, + { + "url":"http://localhost:8001/api/packages/137909", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.73" + }, + { + "url":"http://localhost:8001/api/packages/107591", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.75" + }, + { + "url":"http://localhost:8001/api/packages/137910", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.76" + }, + { + "url":"http://localhost:8001/api/packages/137911", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.77" + }, + { + "url":"http://localhost:8001/api/packages/137912", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.78" + }, + { + "url":"http://localhost:8001/api/packages/137913", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.79" + }, + { + "url":"http://localhost:8001/api/packages/137914", + "purl":"pkg:maven/org.apache.tomcat/tomcat@8.5.81" + }, + { + "url":"http://localhost:8001/api/packages/107123", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M3" + }, + { + "url":"http://localhost:8001/api/packages/129691", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M4" + }, + { + "url":"http://localhost:8001/api/packages/129692", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M6" + }, + { + "url":"http://localhost:8001/api/packages/129693", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M8" + }, + { + "url":"http://localhost:8001/api/packages/100002", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M9" + }, + { + "url":"http://localhost:8001/api/packages/100003", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M10" + }, + { + "url":"http://localhost:8001/api/packages/107188", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M11" + }, + { + "url":"http://localhost:8001/api/packages/129694", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M13" + }, + { + "url":"http://localhost:8001/api/packages/129695", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M15" + }, + { + "url":"http://localhost:8001/api/packages/107295", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M17" + }, + { + "url":"http://localhost:8001/api/packages/107290", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M18" + }, + { + "url":"http://localhost:8001/api/packages/107291", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M19" + }, + { + "url":"http://localhost:8001/api/packages/107305", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M20" + }, + { + "url":"http://localhost:8001/api/packages/107306", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M21" + }, + { + "url":"http://localhost:8001/api/packages/107316", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M22" + }, + { + "url":"http://localhost:8001/api/packages/129696", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M25" + }, + { + "url":"http://localhost:8001/api/packages/129697", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M26" + }, + { + "url":"http://localhost:8001/api/packages/129698", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.0.M27" + }, + { + "url":"http://localhost:8001/api/packages/129699", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.2" + }, + { + "url":"http://localhost:8001/api/packages/129700", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.4" + }, + { + "url":"http://localhost:8001/api/packages/129701", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.5" + }, + { + "url":"http://localhost:8001/api/packages/129702", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.6" + }, + { + "url":"http://localhost:8001/api/packages/129703", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.7" + }, + { + "url":"http://localhost:8001/api/packages/129704", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.8" + }, + { + "url":"http://localhost:8001/api/packages/129705", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.10" + }, + { + "url":"http://localhost:8001/api/packages/129706", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.11" + }, + { + "url":"http://localhost:8001/api/packages/129707", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.12" + }, + { + "url":"http://localhost:8001/api/packages/129708", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.13" + }, + { + "url":"http://localhost:8001/api/packages/129709", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.14" + }, + { + "url":"http://localhost:8001/api/packages/129710", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.16" + }, + { + "url":"http://localhost:8001/api/packages/129711", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.17" + }, + { + "url":"http://localhost:8001/api/packages/129712", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.19" + }, + { + "url":"http://localhost:8001/api/packages/107590", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.20" + }, + { + "url":"http://localhost:8001/api/packages/129713", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.21" + }, + { + "url":"http://localhost:8001/api/packages/129714", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.22" + }, + { + "url":"http://localhost:8001/api/packages/129715", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.24" + }, + { + "url":"http://localhost:8001/api/packages/129716", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.26" + }, + { + "url":"http://localhost:8001/api/packages/129717", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.27" + }, + { + "url":"http://localhost:8001/api/packages/129718", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.29" + }, + { + "url":"http://localhost:8001/api/packages/100220", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.30" + }, + { + "url":"http://localhost:8001/api/packages/109040", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.31" + }, + { + "url":"http://localhost:8001/api/packages/129719", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.33" + }, + { + "url":"http://localhost:8001/api/packages/129720", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.34" + }, + { + "url":"http://localhost:8001/api/packages/72872", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.35" + }, + { + "url":"http://localhost:8001/api/packages/107824", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.36" + }, + { + "url":"http://localhost:8001/api/packages/107836", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.37" + }, + { + "url":"http://localhost:8001/api/packages/137915", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.38" + }, + { + "url":"http://localhost:8001/api/packages/107697", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.39" + }, + { + "url":"http://localhost:8001/api/packages/108025", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.40" + }, + { + "url":"http://localhost:8001/api/packages/137916", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.41" + }, + { + "url":"http://localhost:8001/api/packages/137917", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.43" + }, + { + "url":"http://localhost:8001/api/packages/108092", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.44" + }, + { + "url":"http://localhost:8001/api/packages/108156", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.45" + }, + { + "url":"http://localhost:8001/api/packages/137918", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.46" + }, + { + "url":"http://localhost:8001/api/packages/107713", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.48" + }, + { + "url":"http://localhost:8001/api/packages/137919", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.50" + }, + { + "url":"http://localhost:8001/api/packages/137920", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.52" + }, + { + "url":"http://localhost:8001/api/packages/137921", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.53" + }, + { + "url":"http://localhost:8001/api/packages/108026", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.54" + }, + { + "url":"http://localhost:8001/api/packages/137922", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.55" + }, + { + "url":"http://localhost:8001/api/packages/137923", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.56" + }, + { + "url":"http://localhost:8001/api/packages/107893", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.58" + }, + { + "url":"http://localhost:8001/api/packages/137924", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.59" + }, + { + "url":"http://localhost:8001/api/packages/137925", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.60" + }, + { + "url":"http://localhost:8001/api/packages/137926", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.62" + }, + { + "url":"http://localhost:8001/api/packages/137927", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.63" + }, + { + "url":"http://localhost:8001/api/packages/137928", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.64" + }, + { + "url":"http://localhost:8001/api/packages/100221", + "purl":"pkg:maven/org.apache.tomcat/tomcat@9.0.65" + }, + { + "url":"http://localhost:8001/api/packages/74553", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M1" + }, + { + "url":"http://localhost:8001/api/packages/129721", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M3" + }, + { + "url":"http://localhost:8001/api/packages/74554", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + }, + { + "url":"http://localhost:8001/api/packages/74555", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M5" + }, + { + "url":"http://localhost:8001/api/packages/107825", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M6" + }, + { + "url":"http://localhost:8001/api/packages/107837", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M7" + }, + { + "url":"http://localhost:8001/api/packages/107698", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M8" + }, + { + "url":"http://localhost:8001/api/packages/107699", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M9" + }, + { + "url":"http://localhost:8001/api/packages/137929", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0-M10" + }, + { + "url":"http://localhost:8001/api/packages/107894", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.0" + }, + { + "url":"http://localhost:8001/api/packages/137930", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.2" + }, + { + "url":"http://localhost:8001/api/packages/108093", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.4" + }, + { + "url":"http://localhost:8001/api/packages/108158", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.5" + }, + { + "url":"http://localhost:8001/api/packages/137931", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.6" + }, + { + "url":"http://localhost:8001/api/packages/107714", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.7" + }, + { + "url":"http://localhost:8001/api/packages/137932", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.8" + }, + { + "url":"http://localhost:8001/api/packages/137933", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.10" + }, + { + "url":"http://localhost:8001/api/packages/137934", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.11" + }, + { + "url":"http://localhost:8001/api/packages/108027", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.12" + }, + { + "url":"http://localhost:8001/api/packages/137935", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.13" + }, + { + "url":"http://localhost:8001/api/packages/137936", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.14" + }, + { + "url":"http://localhost:8001/api/packages/107895", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.16" + }, + { + "url":"http://localhost:8001/api/packages/137937", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.17" + }, + { + "url":"http://localhost:8001/api/packages/137938", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.18" + }, + { + "url":"http://localhost:8001/api/packages/137939", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.20" + }, + { + "url":"http://localhost:8001/api/packages/137940", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.21" + }, + { + "url":"http://localhost:8001/api/packages/100240", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.22" + }, + { + "url":"http://localhost:8001/api/packages/142199", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.0.23" + }, + { + "url":"http://localhost:8001/api/packages/100253", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M1" + }, + { + "url":"http://localhost:8001/api/packages/137941", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M2" + }, + { + "url":"http://localhost:8001/api/packages/137942", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M4" + }, + { + "url":"http://localhost:8001/api/packages/108029", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M5" + }, + { + "url":"http://localhost:8001/api/packages/108030", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M6" + }, + { + "url":"http://localhost:8001/api/packages/137943", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M7" + }, + { + "url":"http://localhost:8001/api/packages/137944", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8" + }, + { + "url":"http://localhost:8001/api/packages/137945", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M10" + }, + { + "url":"http://localhost:8001/api/packages/137946", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M11" + }, + { + "url":"http://localhost:8001/api/packages/137947", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M12" + }, + { + "url":"http://localhost:8001/api/packages/137948", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M14" + }, + { + "url":"http://localhost:8001/api/packages/137949", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M15" + }, + { + "url":"http://localhost:8001/api/packages/100254", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M16" + }, + { + "url":"http://localhost:8001/api/packages/100255", + "purl":"pkg:maven/org.apache.tomcat/tomcat@10.1.0-M17" + } + ], + "references":[ + { + "reference_url":"https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html", + "reference_id":"", + "scores":[ + + ], + "url":"https://www.oracle.com/technetwork/security-advisory/cpuapr2019-5072813.html" + }, + { + "reference_url":"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:2.3:a:apache:tomcat:9.0.0:m22:*:*:*:*:*:*", + "reference_id":"cpe:2.3:a:apache:tomcat:9.0.0:m22:*:*:*:*:*:*", + "scores":[ + + ], + "url":"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:2.3:a:apache:tomcat:9.0.0:m22:*:*:*:*:*:*" + } + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json new file mode 100644 index 000000000..4c04506df --- /dev/null +++ b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json @@ -0,0 +1,400 @@ +[ + { + "affected_versions": [ + "8.5.50", + "9.0.30", + "10.0.0-M1", + "10.1.0-M1", + "10.1.0-M16", + "8.5.51", + "8.5.53", + "8.5.54", + "8.5.55", + "8.5.56", + "8.5.57", + "8.5.58", + "8.5.59", + "8.5.60", + "8.5.61", + "8.5.63", + "8.5.64", + "8.5.65", + "8.5.66", + "8.5.68", + "8.5.69", + "8.5.70", + "8.5.71", + "8.5.72", + "8.5.73", + "8.5.75", + "8.5.76", + "8.5.77", + "8.5.78", + "8.5.79", + "8.5.81", + "9.0.31", + "9.0.33", + "9.0.34", + "9.0.35", + "9.0.36", + "9.0.37", + "9.0.38", + "9.0.39", + "9.0.40", + "9.0.41", + "9.0.43", + "9.0.44", + "9.0.45", + "9.0.46", + "9.0.48", + "9.0.50", + "9.0.52", + "9.0.53", + "9.0.54", + "9.0.55", + "9.0.56", + "9.0.58", + "9.0.59", + "9.0.60", + "9.0.62", + "9.0.63", + "9.0.64", + "10.0.0-M3", + "10.0.0-M4", + "10.0.0-M5", + "10.0.0-M6", + "10.0.0-M7", + "10.0.0-M8", + "10.0.0-M9", + "10.0.0-M10", + "10.0.0", + "10.0.2", + "10.0.4", + "10.0.5", + "10.0.6", + "10.0.7", + "10.0.8", + "10.0.10", + "10.0.11", + "10.0.12", + "10.0.13", + "10.0.14", + "10.0.16", + "10.0.17", + "10.0.18", + "10.0.20", + "10.0.21", + "10.1.0-M2", + "10.1.0-M4", + "10.1.0-M5", + "10.1.0-M6", + "10.1.0-M7", + "10.1.0-M8", + "10.1.0-M10", + "10.1.0-M11", + "10.1.0-M12", + "10.1.0-M14", + "10.1.0-M15" + ], + "fixed_versions": [ + "8.5.82", + "9.0.65", + "10.0.22", + "10.1.0-M17", + "9.0.0.M1" + ], + "aliases": [ + "GHSA-6j88-6whg-x687", + "CVE-2022-34305" + ] + }, + { + "affected_versions": [ + "7.0.0", + "7.0.81", + "8.0.0", + "8.0.46", + "8.5.0", + "8.5.22", + "9.0.0M27", + "9.0.0.M1", + "7.0.35", + "7.0.37", + "7.0.39", + "7.0.40", + "7.0.41", + "7.0.42", + "7.0.47", + "7.0.50", + "7.0.52", + "7.0.53", + "7.0.54", + "7.0.55", + "7.0.56", + "7.0.57", + "7.0.59", + "7.0.61", + "7.0.62", + "7.0.63", + "7.0.64", + "7.0.65", + "7.0.67", + "7.0.68", + "7.0.69", + "7.0.70", + "7.0.72", + "7.0.73", + "7.0.75", + "7.0.76", + "7.0.77", + "7.0.78", + "7.0.79", + "8.0.1", + "8.0.3", + "8.0.5", + "8.0.8", + "8.0.9", + "8.0.11", + "8.0.12", + "8.0.14", + "8.0.15", + "8.0.17", + "8.0.18", + "8.0.20", + "8.0.21", + "8.0.22", + "8.0.23", + "8.0.24", + "8.0.26", + "8.0.27", + "8.0.28", + "8.0.29", + "8.0.30", + "8.0.32", + "8.0.33", + "8.0.35", + "8.0.36", + "8.0.37", + "8.0.38", + "8.0.39", + "8.0.41", + "8.0.42", + "8.0.43", + "8.0.44", + "8.0.45", + "8.5.2", + "8.5.3", + "8.5.4", + "8.5.5", + "8.5.6", + "8.5.8", + "8.5.9", + "8.5.11", + "8.5.12", + "8.5.13", + "8.5.14", + "8.5.15", + "8.5.16", + "8.5.19", + "8.5.20", + "8.5.21", + "7.0.84", + "7.0.85", + "7.0.86", + "7.0.88", + "7.0.90", + "7.0.91", + "7.0.92", + "7.0.93", + "7.0.94", + "7.0.96", + "7.0.99", + "7.0.100", + "7.0.103", + "7.0.104", + "7.0.105", + "7.0.106", + "7.0.107", + "7.0.108", + "7.0.109", + "8.0.0-RC1", + "8.0.0-RC3", + "8.0.0-RC5", + "8.0.0-RC10", + "8.0.48", + "8.0.49", + "8.0.50", + "8.0.51", + "8.0.52", + "8.0.53", + "8.5.24", + "8.5.27", + "8.5.28", + "8.5.29", + "8.5.30", + "8.5.31", + "8.5.32", + "8.5.33", + "8.5.34", + "8.5.35", + "8.5.37", + "8.5.38", + "8.5.39", + "8.5.40", + "8.5.41", + "8.5.42", + "8.5.43", + "8.5.45", + "8.5.46", + "8.5.47", + "8.5.49", + "8.5.50", + "8.5.51", + "8.5.53", + "8.5.54", + "8.5.55", + "8.5.56", + "8.5.57", + "8.5.58", + "8.5.59", + "8.5.60", + "8.5.61", + "8.5.63", + "8.5.64", + "8.5.65", + "8.5.66", + "8.5.68", + "8.5.69", + "8.5.70", + "8.5.71", + "8.5.72", + "8.5.73", + "8.5.75", + "8.5.76", + "8.5.77", + "8.5.78", + "8.5.79", + "8.5.81", + "9.0.0.M3", + "9.0.0.M4", + "9.0.0.M6", + "9.0.0.M8", + "9.0.0.M9", + "9.0.0.M10", + "9.0.0.M11", + "9.0.0.M13", + "9.0.0.M15", + "9.0.0.M17", + "9.0.0.M18", + "9.0.0.M19", + "9.0.0.M20", + "9.0.0.M21", + "9.0.0.M22", + "9.0.0.M25", + "9.0.0.M26", + "9.0.0.M27", + "9.0.2", + "9.0.4", + "9.0.5", + "9.0.6", + "9.0.7", + "9.0.8", + "9.0.10", + "9.0.11", + "9.0.12", + "9.0.13", + "9.0.14", + "9.0.16", + "9.0.17", + "9.0.19", + "9.0.20", + "9.0.21", + "9.0.22", + "9.0.24", + "9.0.26", + "9.0.27", + "9.0.29", + "9.0.30", + "9.0.31", + "9.0.33", + "9.0.34", + "9.0.35", + "9.0.36", + "9.0.37", + "9.0.38", + "9.0.39", + "9.0.40", + "9.0.41", + "9.0.43", + "9.0.44", + "9.0.45", + "9.0.46", + "9.0.48", + "9.0.50", + "9.0.52", + "9.0.53", + "9.0.54", + "9.0.55", + "9.0.56", + "9.0.58", + "9.0.59", + "9.0.60", + "9.0.62", + "9.0.63", + "9.0.64", + "9.0.65", + "10.0.0-M1", + "10.0.0-M3", + "10.0.0-M4", + "10.0.0-M5", + "10.0.0-M6", + "10.0.0-M7", + "10.0.0-M8", + "10.0.0-M9", + "10.0.0-M10", + "10.0.0", + "10.0.2", + "10.0.4", + "10.0.5", + "10.0.6", + "10.0.7", + "10.0.8", + "10.0.10", + "10.0.11", + "10.0.12", + "10.0.13", + "10.0.14", + "10.0.16", + "10.0.17", + "10.0.18", + "10.0.20", + "10.0.21", + "10.0.22", + "10.0.23", + "10.1.0-M1", + "10.1.0-M2", + "10.1.0-M4", + "10.1.0-M5", + "10.1.0-M6", + "10.1.0-M7", + "10.1.0-M8", + "10.1.0-M10", + "10.1.0-M11", + "10.1.0-M12", + "10.1.0-M14", + "10.1.0-M15", + "10.1.0-M16", + "10.1.0-M17" + ], + "fixed_versions": [ + "7.0.82", + "8.0.47", + "8.5.23", + "9.0.1" + ], + "aliases": [ + "GHSA-xjgh-84hx-56c5", + "CVE-2017-12617" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_vulnerablecode.py b/vulntotal/tests/test_vulnerablecode.py new file mode 100644 index 000000000..f400df7ea --- /dev/null +++ b/vulntotal/tests/test_vulnerablecode.py @@ -0,0 +1,43 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnTotal software is licensed under the Apache License version 2.0. +# Data generated with VulnTotal require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnTotal or any VulnTotal +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnTotal should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnTotal is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from pathlib import Path + +from commoncode import testcase +from packageurl import PackageURL + +from vulnerabilities.tests import util_tests +from vulntotal.datasources import vulnerablecode + + +class TestVulnerableCode(testcase.FileBasedTesting): + test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "vulnerablecode") + + def test_parse_advisory(self): + advisory_file = self.get_test_loc("advisory.json") + with open(advisory_file) as f: + advisory = json.load(f) + results = [vulnerablecode.parse_advisory(adv).to_dict() for adv in advisory] + expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) From 1cb8ead2702632b77901e8d246a20718ac54abe1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 19 Nov 2022 09:34:44 +0100 Subject: [PATCH 26/36] Use explicit list of datasources This is better, safer and simpler than a discovered list Signed-off-by: Philippe Ombredanne --- vulntotal/datasources/__init__.py | 54 ++++++++++++------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/vulntotal/datasources/__init__.py b/vulntotal/datasources/__init__.py index d39dd560a..eea0cc048 100644 --- a/vulntotal/datasources/__init__.py +++ b/vulntotal/datasources/__init__.py @@ -1,25 +1,11 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import glob import importlib @@ -29,19 +15,21 @@ from os.path import isfile from os.path import join +from vulntotal.datasources import deps +from vulntotal.datasources import github +from vulntotal.datasources import gitlab +from vulntotal.datasources import oss_index +from vulntotal.datasources import osv +from vulntotal.datasources import snyk +from vulntotal.datasources import vulnerablecode from vulntotal.validator import DataSource -DATASOURCE_REGISTRY = {} -files = glob.glob(join(dirname(__file__), "*.py")) -modules = [ - f"vulntotal.datasources.{basename(f)[:-3]}" - for f in files - if isfile(f) and not f.endswith("__init__.py") -] - - -for module in modules: - for name, cls in inspect.getmembers(importlib.import_module(module), inspect.isclass): - if cls.__module__ == module and cls.__base__ == DataSource: - DATASOURCE_REGISTRY[cls.__module__.split(".")[-1]] = cls - break +DATASOURCE_REGISTRY = { + "deps": deps.DepsDataSource, + "github": github.GithubDataSource, + "gitlab": gitlab.GitlabDataSource, + "oss_index": oss_index.OSSDataSource, + "osv": osv.OSVDataSource, + "snyk": snyk.SnykDataSource, + "vulnerablecode": vulnerablecode.VulnerableCodeDataSource, +} From 5033af7f3525d3f9f34e279739a612c241de1921 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 19 Nov 2022 09:39:21 +0100 Subject: [PATCH 27/36] Use standard header and streamline imports Signed-off-by: Philippe Ombredanne --- vulntotal/datasources/deps.py | 28 +++++----------------- vulntotal/datasources/github.py | 27 ++++----------------- vulntotal/datasources/gitlab.py | 31 +++++++------------------ vulntotal/datasources/oss_index.py | 25 ++++---------------- vulntotal/datasources/osv.py | 3 +-- vulntotal/datasources/snyk.py | 28 +++++----------------- vulntotal/datasources/vulnerablecode.py | 25 ++++---------------- vulntotal/tests/test_deps.py | 24 ++++--------------- vulntotal/tests/test_github.py | 24 ++++--------------- vulntotal/tests/test_gitlab.py | 25 ++++---------------- vulntotal/tests/test_oss_index.py | 25 ++++---------------- vulntotal/tests/test_snyk.py | 27 +++++---------------- vulntotal/tests/test_vulnerablecode.py | 25 ++++---------------- vulntotal/validator.py | 27 ++++----------------- vulntotal/validator_runner.py | 24 ++++--------------- vulntotal/vulntotal_cli.py | 27 ++++++--------------- vulntotal/vulntotal_utils.py | 6 ++--- 17 files changed, 87 insertions(+), 314 deletions(-) diff --git a/vulntotal/datasources/deps.py b/vulntotal/datasources/deps.py index e41cb1711..f1c2fc2de 100644 --- a/vulntotal/datasources/deps.py +++ b/vulntotal/datasources/deps.py @@ -1,33 +1,17 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import json + import logging from typing import Iterable from urllib.parse import quote import requests -from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py index a708ebce1..68fe36c1c 100644 --- a/vulntotal/datasources/github.py +++ b/vulntotal/datasources/github.py @@ -1,32 +1,15 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - import logging from typing import Iterable -from packageurl import PackageURL - from vulnerabilities import utils from vulntotal.validator import DataSource from vulntotal.validator import VendorData diff --git a/vulntotal/datasources/gitlab.py b/vulntotal/datasources/gitlab.py index a82ba6c30..00e21f70a 100644 --- a/vulntotal/datasources/gitlab.py +++ b/vulntotal/datasources/gitlab.py @@ -1,28 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import json import logging import os import shutil @@ -33,7 +17,6 @@ import requests import saneyaml from fetchcode import fetch -from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -55,7 +38,9 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: casesensitive_package_slug = get_casesensitive_slug(path, package_slug) location = download_subtree(casesensitive_package_slug) if location: - interesting_advisories = parse_interesting_advisories(location, purl.version, delete_download=True) + interesting_advisories = parse_interesting_advisories( + location, purl.version, delete_download=True + ) return interesting_advisories clear_download(location) diff --git a/vulntotal/datasources/oss_index.py b/vulntotal/datasources/oss_index.py index 845e249a9..df24431ce 100644 --- a/vulntotal/datasources/oss_index.py +++ b/vulntotal/datasources/oss_index.py @@ -1,27 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. -import json import logging import os from typing import Iterable diff --git a/vulntotal/datasources/osv.py b/vulntotal/datasources/osv.py index 97ca7d38e..3610df9d4 100644 --- a/vulntotal/datasources/osv.py +++ b/vulntotal/datasources/osv.py @@ -11,7 +11,6 @@ from typing import Iterable import requests -from packageurl import PackageURL from vulntotal.ecosystem.nuget import get_closest_nuget_package_name from vulntotal.validator import DataSource @@ -27,7 +26,7 @@ class OSVDataSource(DataSource): url = "https://api.osv.dev/v1/query" def fetch_advisory(self, payload): - """Fetch JSON advisory from OSV API for a given package payload """ + """Fetch JSON advisory from OSV API for a given package payload""" response = requests.post(self.url, data=str(payload)) if not response.status_code == 200: diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 3783abd05..145be8560 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -1,34 +1,18 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import json + import logging from typing import Iterable from urllib.parse import quote import requests from bs4 import BeautifulSoup -from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData diff --git a/vulntotal/datasources/vulnerablecode.py b/vulntotal/datasources/vulnerablecode.py index 66b39aa52..238c43bb5 100644 --- a/vulntotal/datasources/vulnerablecode.py +++ b/vulntotal/datasources/vulnerablecode.py @@ -1,27 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. -import json import logging from typing import Iterable from urllib.parse import urljoin diff --git a/vulntotal/tests/test_deps.py b/vulntotal/tests/test_deps.py index a64270061..23821209b 100644 --- a/vulntotal/tests/test_deps.py +++ b/vulntotal/tests/test_deps.py @@ -1,25 +1,11 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import json from pathlib import Path diff --git a/vulntotal/tests/test_github.py b/vulntotal/tests/test_github.py index 1bdc57bff..9221afd24 100644 --- a/vulntotal/tests/test_github.py +++ b/vulntotal/tests/test_github.py @@ -1,25 +1,11 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import json from pathlib import Path diff --git a/vulntotal/tests/test_gitlab.py b/vulntotal/tests/test_gitlab.py index 3ad7162d6..6a8162ce7 100644 --- a/vulntotal/tests/test_gitlab.py +++ b/vulntotal/tests/test_gitlab.py @@ -1,27 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. -import json from pathlib import Path from commoncode import testcase diff --git a/vulntotal/tests/test_oss_index.py b/vulntotal/tests/test_oss_index.py index 5c5dd6854..1681c79e3 100644 --- a/vulntotal/tests/test_oss_index.py +++ b/vulntotal/tests/test_oss_index.py @@ -1,31 +1,16 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import json from pathlib import Path from commoncode import testcase -from packageurl import PackageURL from vulnerabilities.tests import util_tests from vulntotal.datasources import oss_index diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index 1cf8d5a23..4b698e677 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -1,27 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import json + from pathlib import Path from commoncode import testcase diff --git a/vulntotal/tests/test_vulnerablecode.py b/vulntotal/tests/test_vulnerablecode.py index f400df7ea..410be53a2 100644 --- a/vulntotal/tests/test_vulnerablecode.py +++ b/vulntotal/tests/test_vulnerablecode.py @@ -1,31 +1,16 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import json from pathlib import Path from commoncode import testcase -from packageurl import PackageURL from vulnerabilities.tests import util_tests from vulntotal.datasources import vulnerablecode diff --git a/vulntotal/validator.py b/vulntotal/validator.py index ec2f9e395..4601390cd 100644 --- a/vulntotal/validator.py +++ b/vulntotal/validator.py @@ -1,33 +1,16 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import dataclasses -import json from typing import Iterable from typing import List -from vulnerabilities.utils import classproperty - @dataclasses.dataclass(order=True) class VendorData: diff --git a/vulntotal/validator_runner.py b/vulntotal/validator_runner.py index 4a3bf33bb..bdac1cd30 100644 --- a/vulntotal/validator_runner.py +++ b/vulntotal/validator_runner.py @@ -1,22 +1,8 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. diff --git a/vulntotal/vulntotal_cli.py b/vulntotal/vulntotal_cli.py index 667b706df..5353ca4f7 100755 --- a/vulntotal/vulntotal_cli.py +++ b/vulntotal/vulntotal_cli.py @@ -2,33 +2,20 @@ # -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnTotal software is licensed under the Apache License version 2.0. -# Data generated with VulnTotal require an acknowledgment. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnTotal or any VulnTotal -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnTotal should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnTotal is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. import concurrent.futures import json import pydoc -import sys import click + +# TODO: use saneyaml import yaml from packageurl import PackageURL from texttable import Texttable diff --git a/vulntotal/vulntotal_utils.py b/vulntotal/vulntotal_utils.py index bb422c99b..f2a0ebbf6 100644 --- a/vulntotal/vulntotal_utils.py +++ b/vulntotal/vulntotal_utils.py @@ -82,7 +82,7 @@ def parse_constraint(constraint): def github_constraints_satisfied(github_constrain, version): """ - Return True or False depending on whether the given version satisfies the github constrain + Return True or False depending on whether the given version satisfies the github constraint For example: >>> assert github_constraints_satisfied(">= 7.0.0, <= 7.6.57", "7.1.1") == True >>> assert github_constraints_satisfied(">= 10.4.0, <= 10.4.1", "10.6.0") == False @@ -100,7 +100,7 @@ def github_constraints_satisfied(github_constrain, version): def snky_constraints_satisfied(snyk_constrain, version): """ - Return True or False depending on whether the given version satisfies the snyk constrain + Return True or False depending on whether the given version satisfies the snyk constraint For example: >>> assert snky_constraints_satisfied(">=4.0.0, <4.0.10.16", "4.0.10.15") == True >>> assert snky_constraints_satisfied(" >=4.1.0, <4.4.15.7", "4.0.10.15") == False @@ -119,7 +119,7 @@ def snky_constraints_satisfied(snyk_constrain, version): def gitlab_constraints_satisfied(gitlab_constrain, version): """ - Return True or False depending on whether the given version satisfies the gitlab constrain + Return True or False depending on whether the given version satisfies the gitlab constraint For example: >>> assert gitlab_constraints_satisfied("[7.0.0,7.0.11),[7.2.0,7.2.4)", "7.2.1") == True >>> assert gitlab_constraints_satisfied("[7.0.0,7.0.11),[7.2.0,7.2.4)", "8.2.1") == False From 9a559bf273a5060b03d04856e5302d91d1c86a22 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 19 Nov 2022 09:50:29 +0100 Subject: [PATCH 28/36] Use simpler and smaller test fixtures * Do not use files for short lists * Use smaller and discrete test files * Prefer indivisual tests using plain HTML rather than packing data in JSON Signed-off-by: Philippe Ombredanne --- vulnerabilities/tests/util_tests.py | 11 + .../snyk/extract_html_json-expected.json | 218 ------------------ vulntotal/tests/test_data/snyk/html/0.html | 137 +++++++++++ .../test_data/snyk/html/0.html-expected.json | 12 + vulntotal/tests/test_data/snyk/html/1.html | 133 +++++++++++ .../test_data/snyk/html/1.html-expected.json | 12 + vulntotal/tests/test_data/snyk/html/2.html | 121 ++++++++++ .../test_data/snyk/html/2.html-expected.json | 15 ++ vulntotal/tests/test_data/snyk/html/3.html | 137 +++++++++++ .../test_data/snyk/html/3.html-expected.json | 15 ++ .../snyk/package_advisory_url-expected.json | 13 -- .../snyk/parsed_advisory-expected.json | 56 ----- vulntotal/tests/test_data/snyk/purls.txt | 11 - vulntotal/tests/test_snyk.py | 75 ++++-- 14 files changed, 649 insertions(+), 317 deletions(-) delete mode 100644 vulntotal/tests/test_data/snyk/extract_html_json-expected.json create mode 100644 vulntotal/tests/test_data/snyk/html/0.html create mode 100644 vulntotal/tests/test_data/snyk/html/0.html-expected.json create mode 100644 vulntotal/tests/test_data/snyk/html/1.html create mode 100644 vulntotal/tests/test_data/snyk/html/1.html-expected.json create mode 100644 vulntotal/tests/test_data/snyk/html/2.html create mode 100644 vulntotal/tests/test_data/snyk/html/2.html-expected.json create mode 100644 vulntotal/tests/test_data/snyk/html/3.html create mode 100644 vulntotal/tests/test_data/snyk/html/3.html-expected.json delete mode 100644 vulntotal/tests/test_data/snyk/package_advisory_url-expected.json delete mode 100644 vulntotal/tests/test_data/snyk/parsed_advisory-expected.json delete mode 100644 vulntotal/tests/test_data/snyk/purls.txt diff --git a/vulnerabilities/tests/util_tests.py b/vulnerabilities/tests/util_tests.py index 2e7df31ac..b70c6381f 100644 --- a/vulnerabilities/tests/util_tests.py +++ b/vulnerabilities/tests/util_tests.py @@ -42,6 +42,17 @@ def check_results_against_json( with open(expected_file) as exp: expected = json.load(exp) + check_results_against_expected(results, expected) + + +def check_results_against_expected( + results, + expected, +): + """ + Check the JSON-serializable mapping or sequence ``results`` against the + ``expected``. + """ # NOTE we redump the JSON as a YAML string for easier display of # the failures comparison/diff if results != expected: diff --git a/vulntotal/tests/test_data/snyk/extract_html_json-expected.json b/vulntotal/tests/test_data/snyk/extract_html_json-expected.json deleted file mode 100644 index 7babbeeb5..000000000 --- a/vulntotal/tests/test_data/snyk/extract_html_json-expected.json +++ /dev/null @@ -1,218 +0,0 @@ -[ - { - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-2401203": [ - "<1.4.3" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1924465": [ - "<1.4.1" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915648": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915560": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKICMDOCTORPKI-1915390": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKISYNCLIB-1915559": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1915544": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1915643": [ - "<1.4.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORLIB-1583445": [ - "<1.3.0" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORPKI-2401204": [ - "<1.4.3" - ], - "SNYK-GOLANG-GITHUBCOMCLOUDFLARECFRPKIVALIDATORPKI-1915649": [ - "=21.10.0-beta.1, <21.10.8", - ">=22.4.0-beta.1, <22.4.1" - ], - "SNYK-PHP-CENTREONCENTREON-2971034": [ - "<21.4.16", - ">=21.10.0-beta.1, <21.10.8", - ">=22.4.0-beta.1, <22.4.1" - ], - "SNYK-PHP-CENTREONCENTREON-1567260": [ - ">=19.10.0, <20.4.0-beta.1", - ">=19.4.0, <19.4.15" - ], - "SNYK-PHP-CENTREONCENTREON-1536559": [ - ">=21.4.0, <21.4.2", - ">=20.10.0, <20.10.8", - "<20.4.14" - ], - "SNYK-PHP-CENTREONCENTREON-1536560": [ - ">=21.4.0, <21.4.2", - ">=20.10.0, <20.10.8", - "<20.4.14" - ], - "SNYK-PHP-CENTREONCENTREON-1536561": [ - ">=21.4.0, <21.4.2", - ">=20.10.0, <20.10.8", - "<20.4.14" - ], - "SNYK-PHP-CENTREONCENTREON-1534849": [ - ">=19.0.0, <19.4.5", - ">=18.10.0, <18.10.8", - "<2.8.30" - ], - "SNYK-PHP-CENTREONCENTREON-1320017": [ - "<20.4.13" - ], - "SNYK-PHP-CENTREONCENTREON-1320018": [ - "<20.4.13" - ], - "SNYK-PHP-CENTREONCENTREON-1296846": [ - "<21.4.0" - ], - "SNYK-PHP-CENTREONCENTREON-1247370": [ - "<2.8.37", - ">=20.10, <20.10.7", - ">=20.4, <20.4.13", - ">=19.10, <19.10.23" - ], - "SNYK-PHP-CENTREONCENTREON-1075031": [ - "<2.8.37", - ">=21.4, <21.4.1", - ">=20.10, <20.10.7", - ">=20.4, <20.4.13", - ">=19.10, <19.10.23" - ], - "SNYK-PHP-CENTREONCENTREON-570529": [ - ">=0.0.0, <1.6.4", - ">=18.10.0, <18.10.5", - ">=19.4.0, <19.4.3", - ">=19.10.0-beta.1, <19.10.2" - ], - "SNYK-PHP-CENTREONCENTREON-570528": [ - ">=0.0.0, <1.6.4", - ">=18.10.0, <18.10.5", - ">=19.4.0, <19.4.3", - ">=19.10.0-beta.1, <19.10.2" - ], - "SNYK-PHP-CENTREONCENTREON-570527": [ - "<19.10.7" - ], - "SNYK-PHP-CENTREONCENTREON-570051": [ - "<19.4.15" - ], - "SNYK-PHP-CENTREONCENTREON-564443": [ - "<19.10.13" - ], - "SNYK-PHP-CENTREONCENTREON-560860": [ - "<19.4.5" - ], - "SNYK-PHP-CENTREONCENTREON-560859": [ - "<19.4.5" - ], - "SNYK-PHP-CENTREONCENTREON-560847": [ - "<19.4.5" - ], - "SNYK-PHP-CENTREONCENTREON-559334": [ - ">=18.10.6, <18.10.8", - ">=19.10.0, <19.10.2", - ">=19.04.2, <19.04.5" - ], - "SNYK-PHP-CENTREONCENTREON-559444": [ - ">=19.10.0, <19.10.2", - ">=19.4.0, <19.4.5", - ">=18.10.0, <18.10.8", - "<2.8.30" - ], - "SNYK-PHP-CENTREONCENTREON-559445": [ - ">=19.10.0, <19.10.2", - ">=19.4.0, <19.4.5", - "<18.10.8" - ], - "SNYK-PHP-CENTREONCENTREON-559335": [ - ">=18.10.6, <18.10.9", - ">=19.10.0, <19.10.3", - ">=19.04.2, <19.04.7" - ], - "SNYK-PHP-CENTREONCENTREON-551996": [ - ">=19.10.0, <19.10.2", - "<19.4.5" - ], - "SNYK-PHP-CENTREONCENTREON-536206": [ - ">=19.10.0, <19.10.2", - ">=19.4.0, <19.4.5", - ">=18.10.0, <18.10.8", - ">=2.8.0, <2.8.30" - ], - "SNYK-PHP-CENTREONCENTREON-536202": [ - ">=19.10.0, <19.10.2", - ">=18.10.0, <18.10.8", - ">=2.8.0, <2.8.30", - ">=19.4.0, <19.4.5" - ], - "SNYK-PHP-CENTREONCENTREON-535972": [ - ">=18.10.6, <18.10.8", - ">=19.4.2, <19.4.5", - ">=2.7.3, <2.8.30" - ], - "SNYK-PHP-CENTREONCENTREON-473006": [ - ">=0.0.0" - ], - "SNYK-PHP-CENTREONCENTREON-472423": [ - ">=2.8.0, <2.8.28", - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472430": [ - ">=20.10.0, <20.10.3", - ">=20.4.0, <20.4.9", - ">=19.10.0, <19.10.19" - ], - "SNYK-PHP-CENTREONCENTREON-472429": [ - ">=2.8.0, <2.8.28", - ">=18.10.0, <18.10.5" - ], - "SNYK-PHP-CENTREONCENTREON-472428": [ - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472425": [ - ">=2.8.0, <2.8.27", - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472424": [ - ">=2.8.0, <18.10.5" - ], - "SNYK-PHP-CENTREONCENTREON-472422": [ - ">=2.8.0, <2.8.27", - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472418": [ - ">=2.8.0, <2.8.28", - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472371": [ - ">=2.8.0, <2.8.28", - ">=18.10.0, <18.10.4" - ], - "SNYK-PHP-CENTREONCENTREON-472370": [ - "<19.4.17" - ], - "SNYK-PHP-CENTREONCENTREON-469159": [ - "<19.10.0-rc.1" - ], - "SNYK-PHP-CENTREONCENTREON-451340": [ - "<19.10.0" - ], - "SNYK-PHP-CENTREONCENTREON-450214": [ - ">=18.10.0, <18.10.5", - ">=2.8.0, <2.8.28" - ] - } -] \ No newline at end of file diff --git a/vulntotal/tests/test_data/snyk/html/0.html b/vulntotal/tests/test_data/snyk/html/0.html new file mode 100644 index 000000000..976851029 --- /dev/null +++ b/vulntotal/tests/test_data/snyk/html/0.html @@ -0,0 +1,137 @@ + + + + Denial of Service (DoS) in github.com/mattermost/mattermost-server/v6/api4 | CVE-2021-37861 | Snyk + + +
Homepage + About Snyk +

+ Denial of Service (DoS) + + + Affecting + github.com/mattermost/mattermost-server/v6/api4 + package, versions + + <6.0.3 +

+ 0.0 +
+ medium +

  • Attack Complexity

    + + High +

  • Privileges Required

    + + High +

  • Scope

    + + Changed +

  • Confidentiality

    + + High +

Do your applications use this vulnerable package?

+ In a few clicks we can analyze your entire application and see what components are vulnerable in your + application, and suggest you quick fixes. +

+ Test your applications +

  • snyk-id

    + SNYK-GOLANG-GITHUBCOMMATTERMOSTMATTERMOSTSERVERV6API4-2869133 +

  • published

    + 13 Jun 2022 +

  • disclosed

    + 13 Jun 2022 +

  • credit

    + streamer45 +

+ Report a new vulnerability + + Found a mistake? +

+ Introduced: 13 Jun 2022 +

CVE-2021-37861 + Open this link in a new tab +
CWE-400 + Open this link in a new tab +

+ How to fix? +

Upgrade github.com/mattermost/mattermost-server/v6/api4 to version 6.0.3 or higher.

+

+ Overview +

github.com/mattermost/mattermost-server/v6/api4 is an open source, private cloud, Slack-alternative from https://mattermost.org.

+

Affected versions of this package are vulnerable to Denial of Service (DoS) due to improper sanitization of user's password in audit logs when user creation fails.

+

+ Details +

Denial of Service (DoS) describes a family of attacks, all aimed at making a system inaccessible to its intended and legitimate users.

+

Unlike other vulnerabilities, DoS attacks usually do not aim at breaching security. Rather, they are focused on making websites and services unavailable to genuine users resulting in downtime.

+

One popular Denial of Service vulnerability is DDoS (a Distributed Denial of Service), an attack that attempts to clog network pipes to the system by generating a large volume of traffic from many machines.

+

When it comes to open source libraries, DoS vulnerabilities allow attackers to trigger such a crash or crippling of the service by using a flaw either in the application code or from the use of open source libraries.

+

Two common types of DoS vulnerabilities:

+
    +

  • High CPU/Memory Consumption- An attacker sending crafted requests that could cause the system to take a disproportionate amount of time to process. For example, commons-fileupload:commons-fileupload.

    +
    • +

  • Crash - An attacker sending crafted requests that could cause the system to crash. For Example, npm ws package

    +
    • +
+

+ References +

+