From a1ceee6e20163f31d6c555d6a58dfce7440f2d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 7 Dec 2023 17:24:22 +0100 Subject: [PATCH 1/3] [issue-775] catch decoding errors while parsing using the cli tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/clitools/pyspdxtools.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/spdx_tools/spdx/clitools/pyspdxtools.py b/src/spdx_tools/spdx/clitools/pyspdxtools.py index 6a46b0d84..4219c6b81 100644 --- a/src/spdx_tools/spdx/clitools/pyspdxtools.py +++ b/src/spdx_tools/spdx/clitools/pyspdxtools.py @@ -14,9 +14,13 @@ # limitations under the License. import logging import sys +from json import JSONDecodeError +from xml.parsers.expat import ExpatError +from xml.sax import SAXParseException import click from beartype.typing import List +from yaml.scanner import ScannerError from spdx_tools.spdx.graph_generation import export_graph_from_document from spdx_tools.spdx.model import Document @@ -113,6 +117,22 @@ def main(infile: str, outfile: str, version: str, novalidation: bool, graph: boo logging.error(log_string) sys.exit(1) + except JSONDecodeError as err: + logging.error(f"Invalid JSON provided: {err.args[0]}") + sys.exit(1) + + except ScannerError as err: + logging.error("Invalid YAML provided: " + "\n".join([str(arg) for arg in err.args])) + sys.exit(1) + + except ExpatError as err: + logging.error(f"Invalid XML provided: {err.args[0]}") + sys.exit(1) + + except SAXParseException as err: + logging.error(f"Invalid RDF-XML provided: {str(err)}") + sys.exit(1) + except FileNotFoundError as err: logging.error(f"{err.strerror}: {err.filename}") sys.exit(1) From 590bbfa8601a2d4590c6579e20e6fc249c523636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 26 Jul 2024 12:25:58 +0200 Subject: [PATCH 2/3] [issue-806] replace Licensing() with spdx_licensing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- .../jsonlikedict/license_expression_parser.py | 5 +++-- tests/spdx/jsonschema/test_file_converter.py | 10 +++++----- tests/spdx/jsonschema/test_package_converter.py | 14 +++++++------- tests/spdx/jsonschema/test_snippet_converter.py | 10 +++++----- tests/spdx/model/test_package.py | 7 ++++--- tests/spdx/parser/jsonlikedict/test_file_parser.py | 6 +++--- .../jsonlikedict/test_license_expression_parser.py | 5 ++++- .../parser/jsonlikedict/test_package_parser.py | 12 ++++++------ .../parser/jsonlikedict/test_snippet_parser.py | 6 +++--- tests/spdx/validation/test_package_validator.py | 4 ++-- 10 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py b/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py index 29c0ebb99..5a3b545f6 100644 --- a/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py +++ b/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py @@ -2,8 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 from beartype.typing import Union -from license_expression import ExpressionError, LicenseExpression, Licensing +from license_expression import ExpressionError, LicenseExpression +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError @@ -18,7 +19,7 @@ def parse_license_expression(license_expression_str: str) -> Union[LicenseExpres return SpdxNone() try: - license_expression = Licensing().parse(license_expression_str) + license_expression = spdx_licensing.parse(license_expression_str) except ExpressionError as err: err_msg = f'Error parsing LicenseExpression: "{license_expression_str}"' if err.args: diff --git a/tests/spdx/jsonschema/test_file_converter.py b/tests/spdx/jsonschema/test_file_converter.py index 8e6c14049..560ecb1a4 100644 --- a/tests/spdx/jsonschema/test_file_converter.py +++ b/tests/spdx/jsonschema/test_file_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.file_converter import FileConverter from spdx_tools.spdx.jsonschema.file_properties import FileProperty @@ -81,8 +81,8 @@ def test_successful_conversion(converter: FileConverter): spdx_id="spdxId", checksums=[Checksum(ChecksumAlgorithm.SHA224, "sha224"), Checksum(ChecksumAlgorithm.MD2, "md2")], file_types=[FileType.SPDX, FileType.OTHER], - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_in_file=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0"), SpdxNoAssertion()], + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_in_file=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0"), SpdxNoAssertion()], license_comment="licenseComment", copyright_text="copyrightText", comment="comment", @@ -115,8 +115,8 @@ def test_successful_conversion(converter: FileConverter): converter.json_property_name(FileProperty.FILE_NAME): "name", converter.json_property_name(FileProperty.FILE_TYPES): ["SPDX", "OTHER"], converter.json_property_name(FileProperty.LICENSE_COMMENTS): "licenseComment", - converter.json_property_name(FileProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(FileProperty.LICENSE_INFO_IN_FILES): ["MIT", "GPL-2.0", "NOASSERTION"], + converter.json_property_name(FileProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(FileProperty.LICENSE_INFO_IN_FILES): ["MIT", "GPL-2.0-only", "NOASSERTION"], converter.json_property_name(FileProperty.NOTICE_TEXT): "notice", } diff --git a/tests/spdx/jsonschema/test_package_converter.py b/tests/spdx/jsonschema/test_package_converter.py index 9365c8214..c6e63d9f8 100644 --- a/tests/spdx/jsonschema/test_package_converter.py +++ b/tests/spdx/jsonschema/test_package_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.package_converter import PackageConverter from spdx_tools.spdx.jsonschema.package_properties import PackageProperty @@ -123,9 +123,9 @@ def test_successful_conversion(converter: PackageConverter): checksums=[Checksum(ChecksumAlgorithm.SHA1, "sha1"), Checksum(ChecksumAlgorithm.BLAKE2B_256, "blake")], homepage="homepage", source_info="sourceInfo", - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_from_files=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0")], - license_declared=Licensing().parse("MIT or GPL-2.0 "), + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_from_files=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0")], + license_declared=spdx_licensing.parse("MIT or GPL-2.0 "), license_comment="licenseComment", copyright_text="copyrightText", summary="summary", @@ -168,9 +168,9 @@ def test_successful_conversion(converter: PackageConverter): ], converter.json_property_name(PackageProperty.HOMEPAGE): "homepage", converter.json_property_name(PackageProperty.SOURCE_INFO): "sourceInfo", - converter.json_property_name(PackageProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(PackageProperty.LICENSE_INFO_FROM_FILES): ["MIT", "GPL-2.0"], - converter.json_property_name(PackageProperty.LICENSE_DECLARED): "MIT OR GPL-2.0", + converter.json_property_name(PackageProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(PackageProperty.LICENSE_INFO_FROM_FILES): ["MIT", "GPL-2.0-only"], + converter.json_property_name(PackageProperty.LICENSE_DECLARED): "MIT OR GPL-2.0-only", converter.json_property_name(PackageProperty.LICENSE_COMMENTS): "licenseComment", converter.json_property_name(PackageProperty.COPYRIGHT_TEXT): "copyrightText", converter.json_property_name(PackageProperty.SUMMARY): "summary", diff --git a/tests/spdx/jsonschema/test_snippet_converter.py b/tests/spdx/jsonschema/test_snippet_converter.py index a677343b0..36eecd4ac 100644 --- a/tests/spdx/jsonschema/test_snippet_converter.py +++ b/tests/spdx/jsonschema/test_snippet_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.snippet_converter import SnippetConverter from spdx_tools.spdx.jsonschema.snippet_properties import SnippetProperty @@ -72,8 +72,8 @@ def test_successful_conversion(converter: SnippetConverter): file_spdx_id=file_spdx_id, byte_range=(1, 2), line_range=(3, 4), - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_in_snippet=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0")], + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_in_snippet=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0")], license_comment="licenseComment", copyright_text="copyrightText", comment="comment", @@ -98,8 +98,8 @@ def test_successful_conversion(converter: SnippetConverter): converter.json_property_name(SnippetProperty.COMMENT): "comment", converter.json_property_name(SnippetProperty.COPYRIGHT_TEXT): "copyrightText", converter.json_property_name(SnippetProperty.LICENSE_COMMENTS): "licenseComment", - converter.json_property_name(SnippetProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(SnippetProperty.LICENSE_INFO_IN_SNIPPETS): ["MIT", "GPL-2.0"], + converter.json_property_name(SnippetProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(SnippetProperty.LICENSE_INFO_IN_SNIPPETS): ["MIT", "GPL-2.0-only"], converter.json_property_name(SnippetProperty.NAME): "name", converter.json_property_name(SnippetProperty.RANGES): [ { diff --git a/tests/spdx/model/test_package.py b/tests/spdx/model/test_package.py index c533b2812..2f015f1e0 100644 --- a/tests/spdx/model/test_package.py +++ b/tests/spdx/model/test_package.py @@ -6,8 +6,9 @@ from unittest import mock import pytest -from license_expression import LicenseExpression, Licensing +from license_expression import LicenseExpression +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, Package, PackagePurpose, SpdxNoAssertion, SpdxNone @@ -30,7 +31,7 @@ def test_correct_initialization(actor, verif_code, checksum, ext_ref): "homepage", "source_info", None, - [Licensing().parse("license and expression"), SpdxNoAssertion()], + [spdx_licensing.parse("license and expression"), SpdxNoAssertion()], SpdxNone(), "comment on license", "copyright", @@ -57,7 +58,7 @@ def test_correct_initialization(actor, verif_code, checksum, ext_ref): assert package.homepage == "homepage" assert package.source_info == "source_info" assert package.license_concluded is None - assert package.license_info_from_files == [Licensing().parse("license and expression"), SpdxNoAssertion()] + assert package.license_info_from_files == [spdx_licensing.parse("license and expression"), SpdxNoAssertion()] assert package.license_declared == SpdxNone() assert package.license_comment == "comment on license" assert package.copyright_text == "copyright" diff --git a/tests/spdx/parser/jsonlikedict/test_file_parser.py b/tests/spdx/parser/jsonlikedict/test_file_parser.py index e1c6b7a5e..bbe5b70ca 100644 --- a/tests/spdx/parser/jsonlikedict/test_file_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_file_parser.py @@ -4,8 +4,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, FileType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.dict_parsing_functions import parse_list_of_elements @@ -82,10 +82,10 @@ def test_parse_file(copyright_text, expected_copyright_text): "IBM Corporation", ], ) - assert file.license_concluded == Licensing().parse("(LGPL-2.0-only OR LicenseRef-2)") + assert file.license_concluded == spdx_licensing.parse("(LGPL-2.0-only OR LicenseRef-2)") TestCase().assertCountEqual( file.license_info_in_file, - [Licensing().parse("GPL-2.0-only"), Licensing().parse("LicenseRef-2"), SpdxNoAssertion()], + [spdx_licensing.parse("GPL-2.0-only"), spdx_licensing.parse("LicenseRef-2"), SpdxNoAssertion()], ) assert ( file.license_comment == "The concluded license was taken from the package level that the file was included in." diff --git a/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py b/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py index fca36b837..d78a88af3 100644 --- a/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py @@ -16,6 +16,8 @@ [ ("First License", spdx_licensing.parse("First License")), ("Second License", spdx_licensing.parse("Second License")), + ("Apache-1.1", spdx_licensing.parse("Apache-1.1")), + ("Zlib", spdx_licensing.parse("zlib")), ("NOASSERTION", SpdxNoAssertion()), ("NONE", SpdxNone()), ], @@ -34,7 +36,8 @@ def test_parse_license_expression(license_expression_str, expected_license): ( "LGPL-2.1, GPL-2.0, GPL-3.0", [ - "Error parsing LicenseExpression: \"LGPL-2.1, GPL-2.0, GPL-3.0\": Invalid license key: the valid characters are: letters and numbers, underscore, dot, colon or hyphen signs and spaces: 'LGPL-2.1, GPL-2.0, GPL-3.0'" # noqa: E501 + # the error message we receive from the license_expression library somehow cuts off the last license + "Error parsing LicenseExpression: \"LGPL-2.1, GPL-2.0, GPL-3.0\": Invalid license key: the valid characters are: letters and numbers, underscore, dot, colon or hyphen signs and spaces: 'LGPL-2.1, GPL-2.0,'" # noqa: E501 ], ), ("Apache License (2.0)", ['Error parsing LicenseExpression: "Apache License (2.0)"']), diff --git a/tests/spdx/parser/jsonlikedict/test_package_parser.py b/tests/spdx/parser/jsonlikedict/test_package_parser.py index 5b933df8b..209cf5c0c 100644 --- a/tests/spdx/parser/jsonlikedict/test_package_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_package_parser.py @@ -5,8 +5,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import ( Actor, ActorType, @@ -173,17 +173,17 @@ def test_parse_package( ) assert package.homepage == expected_homepage assert package.source_info == "uses glibc-2_11-branch from git://sourceware.org/git/glibc.git." - assert package.license_concluded == Licensing().parse("(LGPL-2.0-only OR LicenseRef-3)") + assert package.license_concluded == spdx_licensing.parse("(LGPL-2.0-only OR LicenseRef-3)") TestCase().assertCountEqual( package.license_info_from_files, [ - Licensing().parse("GPL-2.0-only"), - Licensing().parse("LicenseRef-2"), - Licensing().parse("LicenseRef-1"), + spdx_licensing.parse("GPL-2.0-only"), + spdx_licensing.parse("LicenseRef-2"), + spdx_licensing.parse("LicenseRef-1"), SpdxNoAssertion(), ], ) - assert package.license_declared == Licensing().parse("(LGPL-2.0-only AND LicenseRef-3)") + assert package.license_declared == spdx_licensing.parse("(LGPL-2.0-only AND LicenseRef-3)") assert ( package.license_comment == "The license for this project changed with the release of version x.y. The version of the project included" diff --git a/tests/spdx/parser/jsonlikedict/test_snippet_parser.py b/tests/spdx/parser/jsonlikedict/test_snippet_parser.py index 257a00e3a..3b9b62a19 100644 --- a/tests/spdx/parser/jsonlikedict/test_snippet_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_snippet_parser.py @@ -4,8 +4,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.snippet_parser import SnippetParser @@ -65,8 +65,8 @@ def test_parse_snippet(copyright_text, expected_copyright_text): assert snippet.byte_range == (310, 420) assert snippet.line_range == (5, 23) assert snippet.file_spdx_id == "SPDXRef-DoapSource" - assert snippet.license_info_in_snippet == [Licensing().parse("GPL-2.0-only"), SpdxNoAssertion()] - assert snippet.license_concluded == Licensing().parse("GPL-2.0-only") + assert snippet.license_info_in_snippet == [spdx_licensing.parse("GPL-2.0-only"), SpdxNoAssertion()] + assert snippet.license_concluded == spdx_licensing.parse("GPL-2.0-only") assert snippet.attribution_texts == ["Some example attibution text."] diff --git a/tests/spdx/validation/test_package_validator.py b/tests/spdx/validation/test_package_validator.py index a6ef976ef..d0516fff2 100644 --- a/tests/spdx/validation/test_package_validator.py +++ b/tests/spdx/validation/test_package_validator.py @@ -6,8 +6,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.constants import DOCUMENT_SPDX_ID from spdx_tools.spdx.model import Relationship, RelationshipType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.validation.package_validator import validate_package, validate_package_within_document @@ -45,7 +45,7 @@ def test_valid_package(): ( package_fixture( files_analyzed=False, - license_info_from_files=[Licensing().parse("some_license")], + license_info_from_files=[spdx_licensing.parse("some_license")], verification_code=None, ), "license_info_from_files must be None if files_analyzed is False, but is: [LicenseSymbol('some_license', " From eded3dbf029d4bda3ac9bfbdfca98ddc75e08b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 26 Jul 2024 12:44:18 +0200 Subject: [PATCH 3/3] [issue-815] fix CI for Python 3.7 on MacOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.7 had to be removed completely as even the runner on older macos did not start. Signed-off-by: Armin Tänzer --- .github/workflows/check_codestyle.yml | 3 +++ .github/workflows/install_and_test.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/check_codestyle.yml b/.github/workflows/check_codestyle.yml index 15dfd17f9..92860a527 100644 --- a/.github/workflows/check_codestyle.yml +++ b/.github/workflows/check_codestyle.yml @@ -21,6 +21,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 + - python-version: "3.7" + os: macos-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index 4884b9493..038eb4717 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -18,6 +18,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 + - python-version: "3.7" + os: macos-latest steps: - uses: actions/checkout@v3