From 0a746f3aa5665e87cfc90dc4cd2ce992b8723020 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 17:30:36 +0200 Subject: [PATCH 01/28] Switch to pyproject.toml using flot #3459 We are now using only a pyproject.toml Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-mini.toml | 348 ++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 306 +++++++++++++++++++++++++++++++++++++- setup-mini.cfg | 230 ----------------------------- setup.cfg | 231 ----------------------------- setup.py | 6 - 5 files changed, 648 insertions(+), 473 deletions(-) create mode 100644 pyproject-mini.toml delete mode 100644 setup-mini.cfg delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject-mini.toml b/pyproject-mini.toml new file mode 100644 index 00000000000..fa04cfcfba2 --- /dev/null +++ b/pyproject-mini.toml @@ -0,0 +1,348 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "scancode-toolkit-mini" +version = "32.1.0" +description = "ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts." +readme = "README.rst" +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +requires-python = ">=3.8" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "scan", + "license", + "package", + "dependency", + "copyright", + "filetype", + "author", + "extract", + "licensing", + "sca", + "SBOM", + "spdx", + "cyclonedx", + "package-url", + "purl", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "attrs >= 18.1,!=20.1.0;python_version<'3.11'", + "attrs >= 22.1.0;python_version>='3.11'", + "Beautifulsoup4 >= 4.0.0", + "boolean.py >= 4.0", + "chardet >= 3.0.0", + "click >= 6.7, !=7.0", + "colorama >= 0.3.9", + "commoncode >= 31.0.3", + "container-inspector >= 31.0.0", + "debian-inspector >= 31.1.0", + "dparse2 >= 0.7.0", + "fasteners", + "fingerprints >= 0.6.0", + "ftfy >= 6.0.0", + "gemfileparser2 >= 0.9.0", + "html5lib", + "importlib_metadata", + "intbitset >= 3.0.2", + "jaraco.functools", + "javaproperties >= 0.5", + "jinja2 >= 2.7.0", + "jsonstreams >= 0.5.0", + "license_expression >= 30.1.1", + "lxml >= 4.9.2", + "MarkupSafe >= 2.1.2", + "packageurl_python >= 0.9.0", + "packvers >= 21.0.0", + # use temp advanced patched release + "parameter-expansion-patched >= 0.3.1", + "pdfminer.six >= 20200101", + "pefile >= 2020.1.1", + "pip-requirements-parser >= 32.0.1", + "pkginfo2 >= 30.0.0", + "pluggy >= 1.0.0", + "plugincode >= 32.0.0", + "publicsuffix2", + "pyahocorasick >= 2.0.0", + "pygmars >= 0.7.0", + "pygments", + "pymaven_patch >= 0.2.8", + "requests >= 2.7.0", + "saneyaml >= 0.6.0", + "spdx_tools == 0.8.2", + "text_unidecode >= 1.0", + "toml >= 0.10.0", + "urlpy", + "xmltodict >= 0.11.0", + "zipp >= 3.0.0; python_version < \"3.9\"", + "typecode >= 30.0.1", + # "typecode[full] >= 30.0.1", + # "extractcode[full] >= 31.0.0", + +] + + +[project.optional-dependencies] +# no impact but added for symmetry with the mini config +full = [ + "extractcode[full] >= 31.0.0", + "typecode[full] >= 30.0.0", +] + +# linux-only package handling +packages = [ + "rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux'", + "regipy >= 3.1.0; platform_system == 'Linux'", + "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", +] + +testing = [ + "pytest >= 6, != 7.0.0", + "pytest-xdist >= 2", + "aboutcode-toolkit >= 7.0.2", + "pycodestyle >= 2.8.0", + "twine", + "black", + "isort", + "vendorize >= 0.3.0", + "pytest-rerunfailures", + "bump-my-version", +] + +docs = [ + "Sphinx == 5.1.0", + "sphinx_rtd_theme >= 0.5.1", + "doc8 >= 0.8.1", + "sphinx-reredirects >= 0.1.2", + "sphinx-autobuild", + "sphinx-rtd-dark-mode>=1.3.0", + "sphinx-copybutton", +] + + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + + +[tool.bumpversion] +current_version = "32.1.0" +allow_dirty = true + +files = [ + { filename = "src/scancode_config.py" }, + { filename = "pyproject.toml" }, + { filename = "pyproject-scancode-toolkit-mini.toml" }, + { filename = "pyproject-packagedcode.toml" }, + +] + + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/**/*", +] + +excludes = [ + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src", "tests" ] + +sdist_extra_includes = [ + "etc/**/*", + "docs/**/*", + ".github/**/*", + "samples/**/*", + "tests/**/*", + ".gitattributes", + ".gitignore", + "thirdparty/**/*", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "*.toml", + "*.yml", + "*.rst", + "*.py", + + "requirements*", + + "configure*", + "extractcode*", + "scancode*", + + ".dockerignore", + "Dockerfile*", + # path to executable used by a built app + "PYTHON_EXECUTABLE*", + ".VERSION", +] + + +[tool.isort] +force_single_line = "True" +line_length = 88 +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +skip = "doc,venv,tmp,thirdparty,build,dist" + + +[project.scripts] +scancode = "scancode.cli:scancode" +scancode-reindex-licenses = "licensedcode.reindex:reindex_licenses" +scancode-license-data = "licensedcode.license_db:dump_scancode_license_data" +regen-package-docs = "packagedcode.regen_package_docs:regen_package_docs" + + +# These are configurations for ScanCode plugins as entry points. +# Each plugin entry hast this form: +# plugin-name = f"ully.qualified.module:PluginClass" +# where plugin-name must be a unique arbitrary name for this entrypoint. + +# scancode_pre_scan is the entry point for pre_scan plugins executed before the +# scans. See also plugincode.pre_scan module for details and doc. +[project.entry-points.scancode_pre_scan] +ignore = "scancode.plugin_ignore:ProcessIgnore" +facet = "summarycode.facet:AddFacet" + + +# scancode_scan is the entry point for scan plugins that run a scan after the +# pre_scan plugins and before the post_scan plugins. See also plugincode.scan +# module for details and doc. +[project.entry-points.scancode_scan] +info = "scancode.plugin_info:InfoScanner" +licenses = "licensedcode.plugin_license:LicenseScanner" +copyrights = "cluecode.plugin_copyright:CopyrightScanner" +packages = "packagedcode.plugin_package:PackageScanner" +emails = "cluecode.plugin_email:EmailScanner" +urls = "cluecode.plugin_url:UrlScanner" +generated = "summarycode.generated:GeneratedCodeDetector" + + +# scancode_post_scan is the entry point for post_scan plugins executed after the +# scan plugins and before the output plugins. See also plugincode.post_scan +# module for details and doc. +[project.entry-points.scancode_post_scan] +summary = "summarycode.summarizer:ScanSummary" +tallies = "summarycode.tallies:Tallies" +tallies-with-details = "summarycode.tallies:TalliesWithDetails" +tallies-key-files = "summarycode.tallies:KeyFilesTallies" +tallies-by-facet = "summarycode.tallies:FacetTallies" +license-clarity-score = "summarycode.score:LicenseClarityScore" +license-policy = "licensedcode.plugin_license_policy:LicensePolicy" +mark-source = "scancode.plugin_mark_source:MarkSource" +filter-clues = "cluecode.plugin_filter_clues:RedundantCluesFilter" +consolidate = "summarycode.plugin_consolidate:Consolidator" +license-references = "licensedcode.licenses_reference:LicenseReference" +todo = "summarycode.todo:AmbiguousDetectionsToDoPlugin" +classify = "summarycode.classify_plugin:FileClassifier" + + +# scancode_output_filter is the entry point for filter plugins executed after +# the post-scan plugins and used by the output plugins to exclude/filter certain +# files or directories from the codebase. See also plugincode.post_scan module +# for details and doc. +[project.entry-points.scancode_output_filter] +only-findings = "scancode.plugin_only_findings:OnlyFindings" +ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" + + +# scancode_output is the entry point for output plugins that write a scan output +# in a given format at the end of a scan. See also plugincode._output module for +# details and doc. +[project.entry-points.scancode_output] +html = "formattedcode.output_html:HtmlOutput" +html-app = "formattedcode.output_html:HtmlAppOutput" +json = "formattedcode.output_json:JsonCompactOutput" +json-pp = "formattedcode.output_json:JsonPrettyOutput" +spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" +spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" +csv = "formattedcode.output_csv:CsvOutput" +jsonlines = "formattedcode.output_jsonlines:JsonLinesOutput" +template = "formattedcode.output_html:CustomTemplateOutput" +debian = "formattedcode.output_debian:DebianCopyrightOutput" +yaml = "formattedcode.output_yaml:YamlOutput" +cyclonedx = "formattedcode.output_cyclonedx:CycloneDxJsonOutput" +cyclonedx-xml = "formattedcode.output_cyclonedx:CycloneDxXmlOutput" + + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "bin", + "dist", + "build", + "_build", + "dist", + "etc", + "local", + "ci", + "docs", + "man", + "share", + "samples", + ".cache", + ".settings", + "Include", + "include", + "Lib", + "lib", + "lib64", + "Lib64", + "Scripts", + "thirdparty", + "tmp", + "venv", + "tests/data", + ".eggs", + "src/*/data", + "tests/*/data", + "src/licensedcode/_vendor" +] + +python_files = "*.py" + +python_classes = "Test" +python_functions = "test" + +addopts = [ + "-rfExXw", + "--strict-markers", + "--doctest-modules" +] diff --git a/pyproject.toml b/pyproject.toml index eeb168eb668..0e626dbc3ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,305 @@ [build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "scancode-toolkit" +version = "32.1.0" +description = "ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts." +readme = "README.rst" +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +requires-python = ">=3.8" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "scan", + "license", + "package", + "dependency", + "copyright", + "filetype", + "author", + "extract", + "licensing", + "sca", + "SBOM", + "spdx", + "cyclonedx", + "package-url", + "purl", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "attrs >= 18.1,!=20.1.0;python_version<'3.11'", + "attrs >= 22.1.0;python_version>='3.11'", + "Beautifulsoup4 >= 4.0.0", + "boolean.py >= 4.0", + "chardet >= 3.0.0", + "click >= 6.7, !=7.0", + "colorama >= 0.3.9", + "commoncode >= 31.0.3", + "container-inspector >= 31.0.0", + "debian-inspector >= 31.1.0", + "dparse2 >= 0.7.0", + "fasteners", + "fingerprints >= 0.6.0", + "ftfy >= 6.0.0", + "gemfileparser2 >= 0.9.0", + "html5lib", + "importlib_metadata", + "intbitset >= 3.0.2", + "jaraco.functools", + "javaproperties >= 0.5", + "jinja2 >= 2.7.0", + "jsonstreams >= 0.5.0", + "license_expression >= 30.1.1", + "lxml >= 4.9.2", + "MarkupSafe >= 2.1.2", + "packageurl_python >= 0.9.0", + "packvers >= 21.0.0", + # use temp advanced patched release + "parameter-expansion-patched >= 0.3.1", + "pdfminer.six >= 20200101", + "pefile >= 2020.1.1", + "pip-requirements-parser >= 32.0.1", + "pkginfo2 >= 30.0.0", + "pluggy >= 1.0.0", + "plugincode >= 32.0.0", + "publicsuffix2", + "pyahocorasick >= 2.0.0", + "pygmars >= 0.7.0", + "pygments", + "pymaven_patch >= 0.2.8", + "requests >= 2.7.0", + "saneyaml >= 0.6.0", + "spdx_tools == 0.8.2", + "text_unidecode >= 1.0", + "toml >= 0.10.0", + "urlpy", + "xmltodict >= 0.11.0", + "zipp >= 3.0.0; python_version < \"3.9\"", + "typecode >= 30.0.1", + "typecode[full] >= 30.0.1", + "extractcode[full] >= 31.0.0", +] + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + + +[project.optional-dependencies] +# no impact but added for symmetry with the mini config +full = [ + "extractcode[full] >= 31.0.0", + "typecode[full] >= 30.0.0", +] + +# linux-only package handling +packages = [ + "rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux'", + "regipy >= 3.1.0; platform_system == 'Linux'", + "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", +] + +testing = [ + "pytest >= 6, != 7.0.0", + "pytest-xdist >= 2", + "aboutcode-toolkit >= 7.0.2", + "pycodestyle >= 2.8.0", + "twine", + "black", + "isort", + "vendorize >= 0.3.0", + "pytest-rerunfailures", + "bump-my-version", +] + +docs = [ + "Sphinx == 5.1.0", + "sphinx_rtd_theme >= 0.5.1", + "doc8 >= 0.8.1", + "sphinx-reredirects >= 0.1.2", + "sphinx-autobuild", + "sphinx-rtd-dark-mode>=1.3.0", + "sphinx-copybutton", +] + + + +[tool.bumpversion] +current_version = "32.1.0" +allow_dirty = true + +files = [ + { filename = "src/scancode_config.py" }, + { filename = "pyproject.toml" }, + { filename = "pyproject-scancode-toolkit-mini.toml" }, + { filename = "pyproject-packagedcode.toml" }, + +] + + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/**/*", +] + +excludes = [ + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src", "tests" ] + +sdist_extra_includes = [ + "etc/**/*", + "docs/**/*", + ".github/**/*", + "samples/**/*", + "tests/**/*", + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "*.toml", + "*.yml", + "*.rst", + "*.py", + + "requirements*", + + "configure*", + "extractcode*", + "scancode*", + + ".dockerignore", + "Dockerfile*", + # path to executable used by a built app + "PYTHON_EXECUTABLE*", + ".VERSION", +] + + +[tool.isort] +force_single_line = "True" +line_length = 88 +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +skip = "doc,venv,tmp,thirdparty,build,dist" + + +[project.scripts] +scancode = "scancode.cli:scancode" +scancode-reindex-licenses = "licensedcode.reindex:reindex_licenses" +scancode-license-data = "licensedcode.license_db:dump_scancode_license_data" +regen-package-docs = "packagedcode.regen_package_docs:regen_package_docs" + + +# These are configurations for ScanCode plugins as entry points. +# Each plugin entry hast this form: +# plugin-name = f"ully.qualified.module:PluginClass" +# where plugin-name must be a unique arbitrary name for this entrypoint. + +# scancode_pre_scan is the entry point for pre_scan plugins executed before the +# scans. See also plugincode.pre_scan module for details and doc. +[project.entry-points.scancode_pre_scan] +ignore = "scancode.plugin_ignore:ProcessIgnore" +facet = "summarycode.facet:AddFacet" + + +# scancode_scan is the entry point for scan plugins that run a scan after the +# pre_scan plugins and before the post_scan plugins. See also plugincode.scan +# module for details and doc. +[project.entry-points.scancode_scan] +info = "scancode.plugin_info:InfoScanner" +licenses = "licensedcode.plugin_license:LicenseScanner" +copyrights = "cluecode.plugin_copyright:CopyrightScanner" +packages = "packagedcode.plugin_package:PackageScanner" +emails = "cluecode.plugin_email:EmailScanner" +urls = "cluecode.plugin_url:UrlScanner" +generated = "summarycode.generated:GeneratedCodeDetector" + + +# scancode_post_scan is the entry point for post_scan plugins executed after the +# scan plugins and before the output plugins. See also plugincode.post_scan +# module for details and doc. +[project.entry-points.scancode_post_scan] +summary = "summarycode.summarizer:ScanSummary" +tallies = "summarycode.tallies:Tallies" +tallies-with-details = "summarycode.tallies:TalliesWithDetails" +tallies-key-files = "summarycode.tallies:KeyFilesTallies" +tallies-by-facet = "summarycode.tallies:FacetTallies" +license-clarity-score = "summarycode.score:LicenseClarityScore" +license-policy = "licensedcode.plugin_license_policy:LicensePolicy" +mark-source = "scancode.plugin_mark_source:MarkSource" +filter-clues = "cluecode.plugin_filter_clues:RedundantCluesFilter" +consolidate = "summarycode.plugin_consolidate:Consolidator" +license-references = "licensedcode.licenses_reference:LicenseReference" +todo = "summarycode.todo:AmbiguousDetectionsToDoPlugin" +classify = "summarycode.classify_plugin:FileClassifier" + + +# scancode_output_filter is the entry point for filter plugins executed after +# the post-scan plugins and used by the output plugins to exclude/filter certain +# files or directories from the codebase. See also plugincode.post_scan module +# for details and doc. +[project.entry-points.scancode_output_filter] +only-findings = "scancode.plugin_only_findings:OnlyFindings" +ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" + + +# scancode_output is the entry point for output plugins that write a scan output +# in a given format at the end of a scan. See also plugincode._output module for +# details and doc. +[project.entry-points.scancode_output] +html = "formattedcode.output_html:HtmlOutput" +html-app = "formattedcode.output_html:HtmlAppOutput" +json = "formattedcode.output_json:JsonCompactOutput" +json-pp = "formattedcode.output_json:JsonPrettyOutput" +spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" +spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" +csv = "formattedcode.output_csv:CsvOutput" +jsonlines = "formattedcode.output_jsonlines:JsonLinesOutput" +template = "formattedcode.output_html:CustomTemplateOutput" +debian = "formattedcode.output_debian:DebianCopyrightOutput" +yaml = "formattedcode.output_yaml:YamlOutput" +cyclonedx = "formattedcode.output_cyclonedx:CycloneDxJsonOutput" +cyclonedx-xml = "formattedcode.output_cyclonedx:CycloneDxXmlOutput" -[tool.setuptools_scm] -# this is used populated when creating a git archive -# and when there is .git dir and/or there is no git installed -fallback_version = "9999.$Format:%h-%cs$" [tool.pytest.ini_options] norecursedirs = [ diff --git a/setup-mini.cfg b/setup-mini.cfg deleted file mode 100644 index d9487072a41..00000000000 --- a/setup-mini.cfg +++ /dev/null @@ -1,230 +0,0 @@ -[metadata] -name = scancode-toolkit-mini -version = 32.1.0 -license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft - -# description must be on ONE line https://github.com/pypa/setuptools/issues/1390 -description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts. scancode-toolkit-mini is a special build that does not come with pre-built binary dependencies by default. These are instead installed separately or with the extra_requires scancode-toolkit-mini[full] -long_description = file:README.rst -long_description_content_type = text/x-rst -url = https://github.com/nexB/scancode-toolkit - -author = nexB. Inc. and others -author_email = info@aboutcode.org - -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Topic :: Software Development - Topic :: Utilities - -keywords = - open source - scan - license - package - dependency - copyright - filetype - author - extract - licensing - scan - sca - SBOM - spdx - cyclonedx - -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst - CODE_OF_CONDUCT.rst - cc-by-4.0.LICENSE - -[options] -package_dir = - =src -packages = find: -include_package_data = true -zip_safe = false - -py_modules = - scancode_config - -python_requires = >=3.8 - -install_requires = - attrs >= 18.1,!=20.1.0;python_version<'3.11' - attrs >= 22.1.0;python_version>='3.11' - Beautifulsoup4 >= 4.0.0 - boolean.py >= 4.0 - chardet >= 3.0.0 - click >= 6.7, !=7.0 - colorama >= 0.3.9 - commoncode >= 31.0.3 - container-inspector >= 31.0.0 - debian-inspector >= 31.1.0 - dparse2 >= 0.7.0 - fasteners - fingerprints >= 0.6.0 - ftfy >= 6.0.0 - gemfileparser2 >= 0.9.0 - html5lib - importlib_metadata - intbitset >= 3.0.2 - jaraco.functools - javaproperties >= 0.5 - jinja2 >= 2.7.0 - jsonstreams >= 0.5.0 - license_expression >= 30.1.1 - lxml >= 4.9.2 - MarkupSafe >= 2.1.2 - packageurl_python >= 0.9.0 - packvers >= 21.0.0 - # use temp advanced patched release - parameter-expansion-patched >= 0.3.1 - pdfminer.six >= 20200101 - pefile >= 2020.1.1 - pkginfo2 >= 30.0.0 - pip-requirements-parser >= 32.0.1 - pluggy >= 1.0.0 - plugincode >= 32.0.0 - publicsuffix2 - pyahocorasick >= 2.0.0 - pygmars >= 0.7.0 - pygments - pymaven_patch >= 0.2.8 - requests >= 2.7.0 - saneyaml >= 0.6.0 - spdx_tools == 0.8.2 - text_unidecode >= 1.0 - toml >= 0.10.0 - urlpy - xmltodict >= 0.11.0 - zipp >= 3.0.0; python_version < "3.9" - typecode >= 30.0.1 -# typecode[full] >= 30.0.0 -# extractcode[full] >= 31.0.0 - - -[options.packages.find] -where = src - - -[options.extras_require] -full = - typecode[full] >= 30.0.0 - extractcode[full] >= 31.0.0 - -testing = - pytest >= 6, != 7.0.0 - pytest-xdist >= 2 - aboutcode-toolkit >= 7.0.2 - pycodestyle >= 2.8.0 - twine - black - isort - vendorize >= 0.3.0 - pytest-rerunfailures - -docs = - Sphinx == 5.1.0 - sphinx_rtd_theme >= 0.5.1 - sphinx-reredirects >= 0.1.2 - doc8 >= 0.8.1 - sphinx-autobuild - sphinx-rtd-dark-mode>=1.3.0 - sphinx-copybutton - -# linux-only package handling -packages = - rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux' - regipy >= 3.1.0; platform_system == 'Linux' - packagedcode_msitools >= 0.101.210706; platform_system == 'Linux' - - -[options.entry_points] -console_scripts = - scancode = scancode.cli:scancode - scancode-reindex-licenses = licensedcode.reindex:reindex_licenses - scancode-license-data = licensedcode.license_db:dump_scancode_license_data - regen-package-docs = packagedcode.regen_package_docs:regen_package_docs - -# These are configurations for ScanCode plugins as setuptools entry points. -# Each plugin entry hast this form: -# plugin-name = fully.qualified.module:PluginClass -# where plugin-name must be a unique arbitrary name for this entrypoint. - -# scancode_pre_scan is the entry point for pre_scan plugins executed before the -# scans. See also plugincode.pre_scan module for details and doc. -scancode_pre_scan = - ignore = scancode.plugin_ignore:ProcessIgnore - facet = summarycode.facet:AddFacet - - -# scancode_scan is the entry point for scan plugins that run a scan after the -# pre_scan plugins and before the post_scan plugins. See also plugincode.scan -# module for details and doc. -scancode_scan = - info = scancode.plugin_info:InfoScanner - licenses = licensedcode.plugin_license:LicenseScanner - copyrights = cluecode.plugin_copyright:CopyrightScanner - packages = packagedcode.plugin_package:PackageScanner - emails = cluecode.plugin_email:EmailScanner - urls = cluecode.plugin_url:UrlScanner - generated = summarycode.generated:GeneratedCodeDetector - - -# scancode_post_scan is the entry point for post_scan plugins executed after the -# scan plugins and before the output plugins. See also plugincode.post_scan -# module for details and doc. -scancode_post_scan = - summary = summarycode.summarizer:ScanSummary - tallies = summarycode.tallies:Tallies - tallies-with-details = summarycode.tallies:TalliesWithDetails - tallies-key-files = summarycode.tallies:KeyFilesTallies - tallies-by-facet = summarycode.tallies:FacetTallies - license-clarity-score = summarycode.score:LicenseClarityScore - license-policy = licensedcode.plugin_license_policy:LicensePolicy - mark-source = scancode.plugin_mark_source:MarkSource - filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter - consolidate = summarycode.plugin_consolidate:Consolidator - license-references = licensedcode.licenses_reference:LicenseReference - classify = summarycode.classify_plugin:FileClassifier - - -# scancode_output_filter is the entry point for filter plugins executed after -# the post-scan plugins and used by the output plugins to exclude/filter certain -# files or directories from the codebase. See also plugincode.post_scan module -# for details and doc. -scancode_output_filter = - only-findings = scancode.plugin_only_findings:OnlyFindings - ignore-copyrights = cluecode.plugin_ignore_copyrights:IgnoreCopyrights - - -# scancode_output is the entry point for output plugins that write a scan output -# in a given format at the end of a scan. See also plugincode._output module for -# details and doc. -scancode_output = - html = formattedcode.output_html:HtmlOutput - html-app = formattedcode.output_html:HtmlAppOutput - json = formattedcode.output_json:JsonCompactOutput - json-pp = formattedcode.output_json:JsonPrettyOutput - spdx-tv = formattedcode.output_spdx:SpdxTvOutput - spdx-rdf = formattedcode.output_spdx:SpdxRdfOutput - csv = formattedcode.output_csv:CsvOutput - jsonlines = formattedcode.output_jsonlines:JsonLinesOutput - template = formattedcode.output_html:CustomTemplateOutput - debian = formattedcode.output_debian:DebianCopyrightOutput - yaml = formattedcode.output_yaml:YamlOutput - cyclonedx = formattedcode.output_cyclonedx:CycloneDxJsonOutput - cyclonedx-xml = formattedcode.output_cyclonedx:CycloneDxXmlOutput diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 01a05992e6b..00000000000 --- a/setup.cfg +++ /dev/null @@ -1,231 +0,0 @@ -[metadata] -name = scancode-toolkit -version = 32.1.0 -license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft - -# description must be on ONE line https://github.com/pypa/setuptools/issues/1390 -description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts. -long_description = file:README.rst -long_description_content_type = text/x-rst -url = https://github.com/nexB/scancode-toolkit - -author = nexB. Inc. and others -author_email = info@aboutcode.org - -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Topic :: Software Development - Topic :: Utilities - -keywords = - open source - scan - license - package - dependency - copyright - filetype - author - extract - licensing - scan - sca - SBOM - spdx - cyclonedx - -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst - CODE_OF_CONDUCT.rst - cc-by-4.0.LICENSE - -[options] -package_dir = - =src -packages = find: -include_package_data = true -zip_safe = false - -py_modules = - scancode_config - -python_requires = >=3.8 - -install_requires = - attrs >= 18.1,!=20.1.0;python_version<'3.11' - attrs >= 22.1.0;python_version>='3.11' - Beautifulsoup4 >= 4.0.0 - boolean.py >= 4.0 - chardet >= 3.0.0 - click >= 6.7, !=7.0 - colorama >= 0.3.9 - commoncode >= 31.0.3 - container-inspector >= 31.0.0 - debian-inspector >= 31.1.0 - dparse2 >= 0.7.0 - fasteners - fingerprints >= 0.6.0 - ftfy >= 6.0.0 - gemfileparser2 >= 0.9.0 - html5lib - importlib_metadata - intbitset >= 3.0.2 - jaraco.functools - javaproperties >= 0.5 - jinja2 >= 2.7.0 - jsonstreams >= 0.5.0 - license_expression >= 30.1.1 - lxml >= 4.9.2 - MarkupSafe >= 2.1.2 - packageurl_python >= 0.9.0 - packvers >= 21.0.0 - # use temp advanced patched release - parameter-expansion-patched >= 0.3.1 - pdfminer.six >= 20200101 - pefile >= 2020.1.1 - pkginfo2 >= 30.0.0 - pip-requirements-parser >= 32.0.1 - pluggy >= 1.0.0 - plugincode >= 32.0.0 - publicsuffix2 - pyahocorasick >= 2.0.0 - pygmars >= 0.7.0 - pygments - pymaven_patch >= 0.2.8 - requests >= 2.7.0 - saneyaml >= 0.6.0 - spdx_tools == 0.8.2 - text_unidecode >= 1.0 - toml >= 0.10.0 - urlpy - xmltodict >= 0.11.0 - zipp >= 3.0.0; python_version < "3.9" - typecode >= 30.0.1 - typecode[full] >= 30.0.1 - extractcode[full] >= 31.0.0 - - -[options.packages.find] -where = src - - -[options.extras_require] -full = - typecode[full] >= 30.0.0 - extractcode[full] >= 31.0.0 - -testing = - pytest >= 6, != 7.0.0 - pytest-xdist >= 2 - aboutcode-toolkit >= 7.0.2 - pycodestyle >= 2.8.0 - twine - black - isort - vendorize >= 0.3.0 - pytest-rerunfailures - -docs = - Sphinx == 5.1.0 - sphinx-rtd-theme >= 0.5.0 - doc8 >= 0.8.1 - sphinx-reredirects >= 0.1.2 - sphinx-autobuild - sphinx-rtd-dark-mode>=1.3.0 - sphinx-copybutton - -# linux-only package handling -packages = - rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux' - regipy >= 3.1.0; platform_system == 'Linux' - packagedcode_msitools >= 0.101.210706; platform_system == 'Linux' - - -[options.entry_points] -console_scripts = - scancode = scancode.cli:scancode - scancode-reindex-licenses = licensedcode.reindex:reindex_licenses - scancode-license-data = licensedcode.license_db:dump_scancode_license_data - regen-package-docs = packagedcode.regen_package_docs:regen_package_docs - -# These are configurations for ScanCode plugins as setuptools entry points. -# Each plugin entry hast this form: -# plugin-name = fully.qualified.module:PluginClass -# where plugin-name must be a unique arbitrary name for this entrypoint. - -# scancode_pre_scan is the entry point for pre_scan plugins executed before the -# scans. See also plugincode.pre_scan module for details and doc. -scancode_pre_scan = - ignore = scancode.plugin_ignore:ProcessIgnore - facet = summarycode.facet:AddFacet - - -# scancode_scan is the entry point for scan plugins that run a scan after the -# pre_scan plugins and before the post_scan plugins. See also plugincode.scan -# module for details and doc. -scancode_scan = - info = scancode.plugin_info:InfoScanner - licenses = licensedcode.plugin_license:LicenseScanner - copyrights = cluecode.plugin_copyright:CopyrightScanner - packages = packagedcode.plugin_package:PackageScanner - emails = cluecode.plugin_email:EmailScanner - urls = cluecode.plugin_url:UrlScanner - generated = summarycode.generated:GeneratedCodeDetector - - -# scancode_post_scan is the entry point for post_scan plugins executed after the -# scan plugins and before the output plugins. See also plugincode.post_scan -# module for details and doc. -scancode_post_scan = - summary = summarycode.summarizer:ScanSummary - tallies = summarycode.tallies:Tallies - tallies-with-details = summarycode.tallies:TalliesWithDetails - tallies-key-files = summarycode.tallies:KeyFilesTallies - tallies-by-facet = summarycode.tallies:FacetTallies - license-clarity-score = summarycode.score:LicenseClarityScore - license-policy = licensedcode.plugin_license_policy:LicensePolicy - mark-source = scancode.plugin_mark_source:MarkSource - filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter - consolidate = summarycode.plugin_consolidate:Consolidator - license-references = licensedcode.licenses_reference:LicenseReference - todo = summarycode.todo:AmbiguousDetectionsToDoPlugin - classify = summarycode.classify_plugin:FileClassifier - - -# scancode_output_filter is the entry point for filter plugins executed after -# the post-scan plugins and used by the output plugins to exclude/filter certain -# files or directories from the codebase. See also plugincode.post_scan module -# for details and doc. -scancode_output_filter = - only-findings = scancode.plugin_only_findings:OnlyFindings - ignore-copyrights = cluecode.plugin_ignore_copyrights:IgnoreCopyrights - - -# scancode_output is the entry point for output plugins that write a scan output -# in a given format at the end of a scan. See also plugincode._output module for -# details and doc. -scancode_output = - html = formattedcode.output_html:HtmlOutput - html-app = formattedcode.output_html:HtmlAppOutput - json = formattedcode.output_json:JsonCompactOutput - json-pp = formattedcode.output_json:JsonPrettyOutput - spdx-tv = formattedcode.output_spdx:SpdxTvOutput - spdx-rdf = formattedcode.output_spdx:SpdxRdfOutput - csv = formattedcode.output_csv:CsvOutput - jsonlines = formattedcode.output_jsonlines:JsonLinesOutput - template = formattedcode.output_html:CustomTemplateOutput - debian = formattedcode.output_debian:DebianCopyrightOutput - yaml = formattedcode.output_yaml:YamlOutput - cyclonedx = formattedcode.output_cyclonedx:CycloneDxJsonOutput - cyclonedx-xml = formattedcode.output_cyclonedx:CycloneDxXmlOutput diff --git a/setup.py b/setup.py deleted file mode 100644 index bac24a43d99..00000000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import setuptools - -if __name__ == "__main__": - setuptools.setup() From e45c74b53d94169507551d36a35bdfa30982b7a6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 17:31:02 +0200 Subject: [PATCH 02/28] Introduce new pyproject for packaged code #3459 Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-packagedcode.toml | 158 ++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 pyproject-packagedcode.toml diff --git a/pyproject-packagedcode.toml b/pyproject-packagedcode.toml new file mode 100644 index 00000000000..25f0fd7942f --- /dev/null +++ b/pyproject-packagedcode.toml @@ -0,0 +1,158 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "packagedcode" +version = "32.1.0" +description = "PackagedCode is a library to parse and normalize package manifests, lockfiles and dependencies for all type of app and system packages and report them by PURL." +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +# TODO consider a readme? +# readme = "README.rst" +requires-python = ">=3.8" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "scan", + "license", + "package", + "dependency", + "copyright", + "filetype", + "author", + "extract", + "licensing", + "sca", + "SBOM", + "spdx", + "cyclonedx", + "package-url", + "purl", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "attrs >= 18.1,!=20.1.0;python_version<'3.11'", + "attrs >= 22.1.0;python_version>='3.11'", + "Beautifulsoup4 >= 4.0.0", + "boolean.py >= 4.0", + "chardet >= 3.0.0", + "click >= 6.7, !=7.0", + "colorama >= 0.3.9", + "commoncode >= 31.0.3", + "container-inspector >= 31.0.0", + "debian-inspector >= 31.1.0", + "dparse2 >= 0.7.0", + "fasteners", + "fingerprints >= 0.6.0", + "ftfy >= 6.0.0", + "gemfileparser2 >= 0.9.0", + "html5lib", + "importlib_metadata", + "intbitset >= 3.0.2", + "jaraco.functools", + "javaproperties >= 0.5", + "jinja2 >= 2.7.0", + "jsonstreams >= 0.5.0", + "license_expression >= 30.1.1", + "lxml >= 4.9.2", + "MarkupSafe >= 2.1.2", + "packageurl_python >= 0.9.0", + "packvers >= 21.0.0", + # use temp advanced patched release + "parameter-expansion-patched >= 0.3.1", + "pdfminer.six >= 20200101", + "pefile >= 2020.1.1", + "pip-requirements-parser >= 32.0.1", + "pkginfo2 >= 30.0.0", + "pluggy >= 1.0.0", + "plugincode >= 32.0.0", + "publicsuffix2", + "pyahocorasick >= 2.0.0", + "pygmars >= 0.7.0", + "pygments", + "pymaven_patch >= 0.2.8", + "requests >= 2.7.0", + "saneyaml >= 0.6.0", + "spdx_tools == 0.8.2", + "text_unidecode >= 1.0", + "toml >= 0.10.0", + "urlpy", + "xmltodict >= 0.11.0", + "zipp >= 3.0.0; python_version < \"3.9\"", + "typecode >= 30.0.1", + "typecode[full] >= 30.0.1", +] + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + + +[project.optional-dependencies] +# linux-only package handling +packages = [ + "rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux'", + "regipy >= 3.1.0; platform_system == 'Linux'", + "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", +] + + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/packagedcode/**/*", +] + +excludes = [ + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src", "tests" ] + +sdist_extra_includes = [ + "tests/packagedcode/*", + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "*packagedcode*.toml", + "*.yml", + "*.rst", + "*.py", + + "requirements*", +] From b13df3d5bedf88880162c684920a5e2c1eba1e4e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 17:41:24 +0200 Subject: [PATCH 03/28] Split license data/indexin in new wheels #3459 We now have tow new packages that will be pubslihed as wheels: * licensedcode-data that contains only the plain text files for licenses and license rules * licensedcode-index that contains only the pre-built index with compressed licenses and license rules baked-in Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-licensedcode-data.toml | 62 +++++++++++++++++++++++++++++++ pyproject-licensedcode-index.toml | 58 +++++++++++++++++++++++++++++ pyproject-mini.toml | 14 ++++--- pyproject.toml | 6 ++- 4 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 pyproject-licensedcode-data.toml create mode 100644 pyproject-licensedcode-index.toml diff --git a/pyproject-licensedcode-data.toml b/pyproject-licensedcode-data.toml new file mode 100644 index 00000000000..13fc32a2ad3 --- /dev/null +++ b/pyproject-licensedcode-data.toml @@ -0,0 +1,62 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "licensedcode-data" +version = "32.1.0" +description = "A packaging of the ScanCode licensedb license and license rules database." +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +# TODO consider a readme? +# readme = "README.rst" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "license", + "licensing", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Topic :: Software Development", + "Topic :: Utilities", +] + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/licensedcode/data/*", +] + +excludes = [ + "src/licensedcode/data/cache/*", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src" ] + +sdist_extra_includes = [ + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "pyproject-licensedcode-data.toml", + "*.yml", + "*.rst", +] diff --git a/pyproject-licensedcode-index.toml b/pyproject-licensedcode-index.toml new file mode 100644 index 00000000000..f652f807e55 --- /dev/null +++ b/pyproject-licensedcode-index.toml @@ -0,0 +1,58 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "licensedcode-index" +version = "32.1.0" +description = "A packaging of the ScanCode licensedb license and license rules pre-built index. Can only be used with ScanCode-Toolkit." +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +# TODO consider a readme? +# readme = "README.rst" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "license", + "licensing", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Topic :: Software Development", + "Topic :: Utilities", +] + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/licensedcode/data/cache/*", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src" ] + +sdist_extra_includes = [ + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "pyproject-licensedcode-index.toml", + "*.yml", + "*.rst", +] diff --git a/pyproject-mini.toml b/pyproject-mini.toml index fa04cfcfba2..770d3cb242e 100644 --- a/pyproject-mini.toml +++ b/pyproject-mini.toml @@ -30,6 +30,7 @@ keywords = [ "package-url", "purl", ] + classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -45,6 +46,7 @@ classifiers = [ ] dependencies = [ + "licensedcode-index", "attrs >= 18.1,!=20.1.0;python_version<'3.11'", "attrs >= 22.1.0;python_version>='3.11'", "Beautifulsoup4 >= 4.0.0", @@ -99,8 +101,15 @@ dependencies = [ ] +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + [project.optional-dependencies] +licenses = [ + "licensecode-data", +] + # no impact but added for symmetry with the mini config full = [ "extractcode[full] >= 31.0.0", @@ -138,10 +147,6 @@ docs = [ ] -[project.urls] -Homepage = "https://github.com/nexB/scancode-toolkit" - - [tool.bumpversion] current_version = "32.1.0" allow_dirty = true @@ -193,7 +198,6 @@ sdist_extra_includes = [ "tests/**/*", ".gitattributes", ".gitignore", - "thirdparty/**/*", "*.LICENSE", "NOTICE", "*.ABOUT", diff --git a/pyproject.toml b/pyproject.toml index 0e626dbc3ad..3feb3886972 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ ] dependencies = [ + "licensedcode-index", "attrs >= 18.1,!=20.1.0;python_version<'3.11'", "attrs >= 22.1.0;python_version>='3.11'", "Beautifulsoup4 >= 4.0.0", @@ -104,6 +105,10 @@ Homepage = "https://github.com/nexB/scancode-toolkit" [project.optional-dependencies] +licenses = [ + "licensecode-data", +] + # no impact but added for symmetry with the mini config full = [ "extractcode[full] >= 31.0.0", @@ -141,7 +146,6 @@ docs = [ ] - [tool.bumpversion] current_version = "32.1.0" allow_dirty = true From 98cbc9396d2034edccd6ae0a3a060c0d61e013c0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 17:45:37 +0200 Subject: [PATCH 04/28] Add CHANGELOG for new packaging #3459 Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c0da1d326ac..46b6badfa94 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,12 @@ Changelog v33.0.0 (next next, roadmap) ---------------------------- +- We changed the way licenses text are packaged to help creating smaller packages for publishing + on PyPI. If you relied on actual license text files present in the wheels, you will now need + to install the package scancode-toolkit[licenses] that will also contain the files. However, the + better and preferred way is to run the command line "scancode-license-data" that will create + a directory with all license files, and this works with all builds of scancode-toolkit. + - We now support new package manifest formats: - OpenWRT packages. From f87d920e1883e1b067a22e68b746abcaa5b824c2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 18:16:50 +0200 Subject: [PATCH 05/28] Use proper scripts for release #3459 Remove old mumbo jumbo renaming manifests Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- etc/release/scancode-create-pypi-sdist.sh | 34 ------------------- ...> scancode-create-pypi-wheel-and-sdist.sh} | 4 +-- 2 files changed, 2 insertions(+), 36 deletions(-) delete mode 100755 etc/release/scancode-create-pypi-sdist.sh rename etc/release/{scancode-create-pypi-wheel.sh => scancode-create-pypi-wheel-and-sdist.sh} (89%) diff --git a/etc/release/scancode-create-pypi-sdist.sh b/etc/release/scancode-create-pypi-sdist.sh deleted file mode 100755 index fe8cdd75440..00000000000 --- a/etc/release/scancode-create-pypi-sdist.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -################################################################################ -# ScanCode release build script for PyPI sdists -################################################################################ - -set -e -# Un-comment to trace execution -#set -x - -./configure --dev -venv/bin/python setup.py --quiet sdist - -rm -rf build .eggs src/scancode_toolkit*.egg-info src/scancode_toolkit_mini*.egg-info -cp setup.cfg setup-main.cfg -cp setup-mini.cfg setup.cfg - -venv/bin/python setup.py --quiet sdist - -cp setup-main.cfg setup.cfg -rm setup-main.cfg - -venv/bin/twine check dist/* - -set +e -set +x diff --git a/etc/release/scancode-create-pypi-wheel.sh b/etc/release/scancode-create-pypi-wheel-and-sdist.sh similarity index 89% rename from etc/release/scancode-create-pypi-wheel.sh rename to etc/release/scancode-create-pypi-wheel-and-sdist.sh index 5ab2fe8e988..70add72070b 100755 --- a/etc/release/scancode-create-pypi-wheel.sh +++ b/etc/release/scancode-create-pypi-wheel-and-sdist.sh @@ -9,7 +9,7 @@ # ################################################################################ -# ScanCode release build script for PyPI wheels. +# ScanCode release build script for PyPI wheels and sdist. # Build a wheel for the current Python version ################################################################################ @@ -22,7 +22,7 @@ venv/bin/scancode-reindex-licenses python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" ) -venv/bin/python setup.py --quiet bdist_wheel --python-tag $python_tag +venv/bin/flot setup.py --quiet bdist_wheel --python-tag $python_tag rm -rf build .eggs src/scancode_toolkit*.egg-info src/scancode_toolkit_mini*.egg-info cp setup.cfg setup-main.cfg From 00b5f77bebd5f2b179d7ee70d9c61ad5f8fe5cce Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 18:17:45 +0200 Subject: [PATCH 06/28] Add README.rst to license data packaging #3459 Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- src/licensedcode/data/README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/licensedcode/data/README.rst diff --git a/src/licensedcode/data/README.rst b/src/licensedcode/data/README.rst new file mode 100644 index 00000000000..b1aca2a83c5 --- /dev/null +++ b/src/licensedcode/data/README.rst @@ -0,0 +1,8 @@ +ScanCode license data +===================== + +ScanCode license detection is about finding common texts between the text of a query file +being scanned and the texts of the indexed license texts and rule texts. + +This package contains the data (possibly compressed and pickled depending on how it is packaged) +of all rules and licenses used by ScanCode From 1cd163b15f4b9e4976508d711f71efa0ae51afc5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 18:18:22 +0200 Subject: [PATCH 07/28] Improve license data packaging #3459 Actually make this works for building with flot. Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-licensedcode-data.toml | 9 +++++---- pyproject-licensedcode-index.toml | 10 +++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyproject-licensedcode-data.toml b/pyproject-licensedcode-data.toml index 13fc32a2ad3..67c4ba03679 100644 --- a/pyproject-licensedcode-data.toml +++ b/pyproject-licensedcode-data.toml @@ -6,9 +6,9 @@ build-backend = "flot.buildapi" name = "licensedcode-data" version = "32.1.0" description = "A packaging of the ScanCode licensedb license and license rules database." +long_description_content_type = "text/x-rst" +readme = "src/licensedcode/data/README.rst" license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } -# TODO consider a readme? -# readme = "README.rst" authors = [ { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, ] @@ -33,11 +33,12 @@ Homepage = "https://github.com/nexB/scancode-toolkit" wheel_path_prefixes_to_strip = [ "src" ] includes = [ - "src/licensedcode/data/*", + "src/licensedcode/data/**/*", + "src/licensedcode/README.rst", ] excludes = [ - "src/licensedcode/data/cache/*", + "src/licensedcode/data/cache/**/*", ] metadata_files = [ diff --git a/pyproject-licensedcode-index.toml b/pyproject-licensedcode-index.toml index f652f807e55..159b351ccd9 100644 --- a/pyproject-licensedcode-index.toml +++ b/pyproject-licensedcode-index.toml @@ -6,9 +6,9 @@ build-backend = "flot.buildapi" name = "licensedcode-index" version = "32.1.0" description = "A packaging of the ScanCode licensedb license and license rules pre-built index. Can only be used with ScanCode-Toolkit." +long_description_content_type = "text/x-rst" +readme = "src/licensedcode/data/README.rst" license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } -# TODO consider a readme? -# readme = "README.rst" authors = [ { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, ] @@ -33,7 +33,11 @@ Homepage = "https://github.com/nexB/scancode-toolkit" wheel_path_prefixes_to_strip = [ "src" ] includes = [ - "src/licensedcode/data/cache/*", + "src/licensedcode/data/cache/**/*", + "src/licensedcode/data/cc-by-4.0.LICENSE", + "src/licensedcode/data/scancode-data.ABOUT", + "src/licensedcode/README.rst", + "src/licensedcode/data/README.rst", ] metadata_files = [ From 286c272a487efe9dc600afe2523f217065acb3ba Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 19:03:11 +0200 Subject: [PATCH 08/28] Ensure flot is always available #3459 Since we use pip with --no-build-isolation we need to provide flot before we install anything. Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- etc/thirdparty/flot-0.7.2-py3-none-any.whl | Bin 0 -> 21905 bytes .../flot-0.7.2-py3-none-any.whl.ABOUT | 2 ++ requirements-dev.txt | 1 + 3 files changed, 3 insertions(+) create mode 100644 etc/thirdparty/flot-0.7.2-py3-none-any.whl create mode 100644 etc/thirdparty/flot-0.7.2-py3-none-any.whl.ABOUT diff --git a/etc/thirdparty/flot-0.7.2-py3-none-any.whl b/etc/thirdparty/flot-0.7.2-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..c74b7b9af51ece90bae1111f15ae12c489dc888e GIT binary patch literal 21905 zcmZ6yLy#~^6Sdjyw{6?DZQHhO+qP}nwr$(CJ@=o9iI{J;wWx?(t-VK$nwHhJD6;Qb9XmO4 zv%ZB>0kZT;)KxJ9Ohg5eCb`J8q*#@NC?*3_V^8w$TW*K=rjAWcyEC4(#q1AP+d|Sg z#J0BHZ*K1ai7Vz(6(rY?RqQ1;tu5^mngzvre!{zN;*acujS)VD+Wf%}sJ4p5S`)rP9Z0W%K)<~K9_m)#mk3h&q*l;s|LlEOW)N- zC+c=}j&aa+Y7|a$W8f2tVpVOR*gmGWM(Ra6=(ly zYf&K$;aL#{1$j*t4=v=?y|N9q4JXc6a^Pp&$9S8C?gBAo2?giskwSG5)>TZ+(XQY{ zx2(PtoZ4BU{Jg0W!ViYJjW|oLX62%_*u?7B^+uub*IjHRP=9CZr{om3l>+iAO34JZ z9QMGU5evzXvSTqxNpDu1ESQW6_*TdSn8TxIMeSfWOe7jt#( z<|eG!6t~$;Y}9QIBqs*Nmw+nM^h%DPclilB?SJd~@D=`R{SsxxviXmo-DCtk@t*M7 zR%Sd<)wm(GPi}Pfs+)?_MlziZ^p^c|`HlylnIHXSjP(vKh{beHF@f15mI!MqFj$tF z|9siE7ncddW`gs*sCUk}o%TI}WYD;zryCkGscf)~+R`E;w&s{Q%MdEkUC7RhKyWPz zNto8BS*NolDP;haSU3DawDDAq&j|1YZUq&1Y6G(0Dh#$n{@5iQq#tgCYJLfC%*RWr zw{YI#STCP$kouf9%n0|zTp8VGtP6lwSyX~`iV9>(A^8km1K5wAS$UfTLK|f1h_as~ z+$XQvTemLidPwfwzVjcO2WRSclsfZ>@J^VY?0B{H{yra1Ny(e@=keh@clfF_x^58M z>y5AiMAW$=`n8Zh+JkBJbiLmcw8Z9 z80PkGI!ZNtkBG^Nt4mfN{r#U$|XbQSc!k1q16&hg~`M`Ra&6Mu&OkK_XWkI32>SlIsGB5M=NV>LvN z9_;oLNu+^TUejC>MG%!m!Yo99(+MHsiPNL+gFjL5`xevG%7)`MH&8Qzc{VrX=42>T zAn(+SObI95A>_x)To;4!qI^EvGsbMz+Zh?NK{ziLXGBlqTu;xiJe^iCs{NZ!*LBuL z=G}4bGBCzI!#4R>G*6uz&oPdpnRA>XWN9*_rC?_x@B4T0RsRyLjQjXHgvJD zHa4)g`0rJ@QPr{9W=HmUtAoFd0=Jf1Qfvh|C9nX3kx82e(a-rHNohKGY3m)2`=-9WACZ7|hY)`e`F(+7oeO{R=Vk79F6&6AHkY~Mt8TU~w zQx*^fAq!b;;nbyNlSZ52loe^I-pZtXO1vm(q4wXcZzM?>hu)ZeQdfbhfI#}upjgq{lZ=ltF45;!e|SM`H}jWUYcClO#v zOtRZ!sPgh|udT*~cF4Qw|3h$^BUO+pXILn{1alxuIKUU8qp+37N(4_FGVK|%Sjr!- z(^=Y^(hff-_eIG9A!6Dp?HaY8B5U094LbRI*)lv!>V{%zLVZ_Y-v171X~H1Mw7q3( zi2q0MI|%Z8_39$*f#o2LFg7oGXNdKBZa!xV`Xq+dT?dx=tc547Gs2^PFc`fRcOpcz z9RChp`@S1HU1I1zd`f{WI7S{G3DIjGEuUYRcTSuhayi)Kooa5)GAa>$)l5SUNhWDP zZMi?dZ+}Wl{HvU6=o7ZnpVsCp?q4=YeLrdEjN3Dkn8kv%vjxuuiMBmPMeHH=ORuXs z0ZZCM9^zkyWG4V1nKpm>e$$GS@DX4QTYaqSM;(K*1ZvBQ%QjaH8 zWeq;D3v9`dkTb?yl9XF9*?!4G$Z&)oH(v4j{YN~UweIXN3TI5u>>_K*;s*_$ZtlEe zp7dqerF$fL3tP;kElp-*Y>Z;K8sD4X=lOWe!=R>jyV7F4A5qijhXnNHwSV@}v1I^* zQ($3(v+yUsb%X78#SG@+@`?7p&B|tik+ui}06@|5|lM)64F- z72(g1541?=me^SBwP(=~5KYFqjN=%0bR!M52iI9UqH9W>s*=#e>xlh)?R@Qu>|#1T znf&Gx@i_!@LW3}D=-{4B?qlE>kc*G&W9hnNTBKy;T7-a-C{2;s>GkDf;`)M`&Z7!+ z9;eE4YkXz1o?T9z#J1Ozu9DlNn^L`&a%o(|O1a2QJP(`Km)8W%s^uWlN3C^>YT;Vz zg=MW!ztYgk$dZSTU0tHd7R{oT75t_36nsiC6IinY^;s!zOn1AWiOmsiWEtHlC9Fflm=b0 z5N2=La}u)Q*v2AVA%BZZ&NS?OZ{#3}<7e#94Q&j9x<_4VhV@&~{IK5KM5BK|V6$-M zq!5|Hs|<7|r_bZ<^_`jP65zy=jQ(6IBkt%=B9!bnAI5u&Djc!kSeWoc2txk51uGiBAOxoDQ zutX_djI~mbXJjF^U15+Pb)Ds=I4a3F5B=x6_eb7_Z_<-OYG7D~#<2*)Om9|NE6or# zAKV7teJrzF{|%PU@&TAy`X=D_`!ISv2>T&;01mPPr8~pb4vRj>*Q=r^G2MDAOaAHmb;;YJ3)>#30;m)W9kQ*{y2dX}pq)3P`R$#8KB%)gOtjmaT*uJubDh;>2{{>TFm@T@Z2{T z#6O{EtrGFaegdr!dNlDvKW@1ROawg6y?4Ejj6%;`B@p${c#%o9su@1Ybp zN_L*3>m@Fa#Z3t{M%CjXV3rRt2MuWShba233OM2z5x^vUxe>@+r6Zh+rlCD~QyK%1 z5yIJ@4<=22u8ggt97cQpo?EpOZ3TtSWFe=t>zKo40;3TeVW#!gO@?2d6)W4z{nFSS z1?P!~%52WuepFToShp+e%{hn2tiS_EkaFntX|B! zV&q$`8hTj_#267>=$vi6mThD&D6US#s5Qr+u;B__4dw={pC&TeGyuUpw83QDvf|Lh z_yzzdI5u(O)tvCSyEjENtPyN``3{R9!*u zVykEx9$Yuv1Qyq=MB`AQ(gn}Fak>%=KLoq8T>ddfl>d1vvI;jcX@?UUFh{`BGhKmF ze8%(YAU+pTpS4oY$2Af>5CaMK;gbHk-DjWZMf@F&2vI+s@gL4l-tKqTr(4*U-@+fT zqZ#_cM3lLAqz^tZQPwL0uAUb|&VodXfiwvaZC3yz!!?KLEz7{Fad!f-no@-mz02+yxodFM9cwd zrw%Pa#@BVYh(*r~IKqaBO5MsEs~w*W=F0HiQzOdrdA0?NFit}kZrh=*;qzpxqCJbi zwJ%vwH;l#li)Q<@6KG_uH_pJkPe5OAGpAlt(-PC`^~OQ+tSb4mBpZI@>A&n{!b6$? z3-Y6ZdvDgWFewzZ?m%(vjc%N_Bq_r~x3u%%h@>M}66kREnJmbXz^LIv!u(OM8IKam zjHkuBB8EB9A%{mV2?%e`@j}Q{E=SW5DR50EbjK^ts>W63&N)EZLL*&`_mebllKNq5 z0tHcxg5sbh7?r$O#C|ce5`br;`4kk>Ej%W_0S&MKvQCM%i?{_b?n%@{uQ5I!o$;?2+kE3ih|CZFXhPZ66B74orGzn zVg&$zJK6;J?~-dy_J`faeKcbcgq{*Xn>{C?4vRQrHCFbqKTT%4wDUpW1wnI{O_%*n z+hPoVsWEYQfa^H%0l~=SZFDRJ0T(LX5JKpb-%j#A=jA0GT;nJ}$-_Id@ zG%Ont6K%dK0e;H*46+*bNKrTIpBIUHbOfXaCn0XaMAe2D3;a=ciSF&d}dJz z>pdHUOwX#Ab@M8zJU*;d+xFgN3a7D!butlFRl#Qsm3lIkb$(0KFr(Hk@f;DD*H?*p zKK-*n_$&T-3aW+_sEI+%+WYQep>g@1-+u44Zq0~5_x_j71ESNrke^{#6nmL(-QT&+$VQL!66QAfIEhGN&U9UTuYNOrsP;Iw zvNpi>+&_!o1e%Plf=to&gTOqo0CvDkq$?#nxF+PHOj|GklB_MD#WZc=?3xXUK%ep ziv`HD_kbIZVvU=j959>0$X6NgK>EMd-AdgI`67b30D>6&q_MgG=rQ-key10wcYAy| z3}LHAr)}I}*1;9tkg(uau9M3{z2+y4-xVU=!#I|DN*T$m*U`GD`t%JZmekP8?W~I4 z=dimNmui(nc5tM558iA?0`!U~E{%|a-Eit_BLbhz{aU|;%e%k`vIkd}^po2Epu0TH z5#UF|i)4hU2-PP=jex5QY8l2fK_Ydb(}XRc^awipmg;5DtwT3J7*u91_2(TIz@an* zm@^M{laceATG#CF0h~vz)Omw`%xp-Uq#Jh@Y&P%a^o(iGEc$;pMYe_S%rrg^ps+5> zrdb>$dl$v{A8%ixj@Zb`BIIDN32b1S`ramEJ8@OTp(ldxew}UFe^nv*#D5>~{Jy%R z{2|FDLt+Opfw~8-Uu)^o!@#ff_XULH2dWy~AzId!TpSAOgqzah{u9lZ|Ef4%mmT(9PeQuGoUK zX6j2p0sIvN^J??kNVD|u8CfomYJyok#HH!u#xDJJ$j~jc+70OnL<*U3=SMUtY9Ix4 z?U#H}MSZb4T+i+xuzFhsoN9AO)&i?t$X;`8^tX-FRyaVG_C-8A4g~8ijUgj9kY5|f z!@fQod1=x&#RS9Q@iA_No<}mq?1eToVD8fWxXc4Zidt?P$B;MO9>wwS-;-p5-9s@G zv-m3DDF?wV*x_hiL)xF?D{r%o_!Nv(u?}C;nyt8ROd=bFKZBSs<{>4lGLi{iN zE%3KhG5?qRXYl?H{n?sYnEjvlAImCkYxJSZcO+HRGkNGJ=VsZ0Rwank6_7~4npjqg zAN8N4w#2jov5;b~65QJ!7tZVrJDlY<+GvA z@eb(B_T#gWF7|ruWc<1{ID4db#pv$9kugxe8WNb_qm9}PBtYi<7D^r0 zYv^vTZDu8g|A}~fAez)LegSLUWTT?L$;$O2;UAsc@w_q6c0(BJwQe&5sQybX*zfwC-5sTaDn=F7h>1=o8#wz}Gn zgHlVzAL;3Y+Do;s()m76l~0GJxjC;o{*|mD51j$YnuF3)66`>g*_W=!bg8JP-F=>M znj@>wYtsAznHMfS7yPcku-l{Sv&v(|eF6CSzP^5H2k2IBmzho)vw&QbBEf954;Hxe zsK^QMP`HMm25|PPzVggpNTMhCGB0fDBj*j(k=~<&-PbE8A{@zNpRH4B^ zHwb523Gg@Yw5hDv(5trKuNGrn$PZlZPb~k`q@nlQU8(6Zq+;`f)a9O1xyBrErb*5N zN$aY3s-ehl<3qjx)=3MboH-DAk|N1uYY8jKf;vbo2}TI$YoaQhm_Y&-B7+6gR7Bt# zy1pvhMdcr=(Mcc=Zh)}~3egW@AAID22U`4KpFd2r80Y^Q? z+-%rcm;nnw%y;+opd2qdNt;D5Hrz=xG2>g<*BP}DcHXll@ zg;;PkI{zzKaJ`fb{&X&=OpeYle1|r-kow-BD_{=_&{m+6sSjInxI82^oMlI<95aqO zBrUje@^&lW*I~X2w$yXYwV-g|A<-F|OOv-U7EeS=Emi3!5aUxCzx;$4p)4WQ4lJ^>_A3G9N2VOA5&eeSDCqLPvmMA;RvTw?x!>r{+R?o zg>rw@#XT@{7Ah;l23)d*fiT^Z#eD(A(9KvsGy;8ye@MEw!gPfU;J0eGbWMF8w{upS zO?s~C9o-zCcJKSAonId$ueW-(1giXIysm&Xeh>h% zl?;ZWWu6#(sVHt6w~cCisO-+W54nUH`rqtrv;4+RUZxjGIcfvbaBi(W{HywW1Y?n= zvWj}qQj>d(<{}JQKi?*|cm-I{Qp8s=o&x0gepPGi1)IqRDnNalkq1?ZJDyD=_dljHjIz%}ayk#QI**|NLu1a$;G+Y7bpdg|aoBb=-$=0qUU+}l z)XOkWAOD2*3-on_$1lLJJWn(2JL60f%V1U}!$%4MyPH+8>jdA3{+(_QfK6!d$SZOE zfQ=HCJ$V4f;{Nz?3$=;BlD{b>vB)pW|8;mISIVFssN@s-A*TId3`GL zxIir|)AL`O-kX%MLA-A+Ody`}6`s#+_3@%(+HO8_%%$&#j#};5Fmcmb*d(y9Au5`WAqJ^H{hEUT}3Ds@i_65CLG0f7wNp zQ&Th2E4>XHO}AbfSs-VIvO+75@@bsqZqKRYV-ed7`rM_qXjwDDTU5S3Pg`qzeSY-D z?k2DN%R%^|&toyHiucw6b5jDI^97R-NU%3Uxfwg%vTY_qcvSO zrAOuUtzDI@Cb2wHF?BtLdis|b<(T{Ia7!2Hd+St)kn+u`!e8*N$wkWKi}I9ghixpu z>JgU%5MIE>6GDxp*ryUQ>n-tyW+wOh$De2bPE+Xy)_7@8>}J2%$Jca9TF-bY~Ug?09up z@DQfU=Db?>Nr9Q_unw<(?1gy<1e~sWVs(b-Uw79z_ZQX_oF`qq;z{#OVhCE1Ew6fu zq+zW0PP-2|3(2RR+q57pPt|futQOePilQ$1REPIJh+9Q0Nlv{ zjOg1n9?Y`Fu9GO}7b+|sok#BEh+IpzNMI)`?Vt;LRCa-pMJda^V^>^$!SD`L3MAj` zD~ggA*BDTXUTYzeF&TB0%;}GJ6dV{;?%eh3<5Gt-lu<*YJsO6Exg)4lTp-TAo2OfA z05edFUvylp;Qz@_^rRnSxrS3jrx&foQWIQ-{lttmC(cHB#SZm3fZIhp6eE0;zL>9r(^8tqn?Z4 zw%5Vh3N!)Ti?#=K%4D<1y^z~@$9hwUVJ+uD8+(}hVcaCyCN|q*+b6*wxEi7 z#E%wLr)z%Bw)l^Ut5&wm9HOjjG^@ifEExbi1^4)UUEZMVQjH%JcZAndLz!wI6IL8g zD1rVh%b?p3^;1Kom8zpKdmdE$%sJ8{( zDiQf2oKjcWVnK_1=MSVqiJ}fw)INxd)`aGffp-`wf>e&XkD$Jkoln^l%n{Nek|iP`U@OL3Zj)Xt@J>F6Q??{0vcuk@gJfDI5@!=*XL*S^ zoF712=157bhn&1>Q>UIOx8a0^^T2B9#US^NfoL51EYd*rlm@bN@!89+HaozXa4#a* zJ)6rBr|}*}GW(iP>#4*C1~#{qqbRE}U6}ntU z)|(`Cp^&d2A+0O6#MZN|2dUph31&kr$~ycY!f|@Ned2h19Af3e;XmTke<8psSQKoj zvOyY1>98X!B|T8Lg4$G9w@3^%Yi@-olqc+9SPzq7Y!ZpC5}KP^c!}C1yLSdzlji8l z$+Bo>+Ke^gt6cUzNPsKq$)ZZ!_LC2PVg7iLJC$XsxeS=F=AY7*u3o;roF9)+ zGkU3LU_bXnxb|_AbB+%dZB`oS5UKH}n;tT6p6VVRmM!vHFt$;&BVnLNpio0L zxIyUdP$`}4{z2dN>@`Q96aD*f|2x<9yRWkD%)FmbCbF)gN>iUl0~cb(oc$y#==3QJ{U_r*%Db%Ck6vh8HCPj90J?z^f?LrnExFHNjDB^I1}>mDoKV7CXUf0gr^FgsNforER--2dhaXQV{(SpNV3uCM?A{u?s?KOYR8j4hm;|BDw|H1zDY*b)8S%JwVP zgFx6F-)y^}K+3vgI=6cCvq;xL)_su7qf$4-3k4J3efD|4GGNYmx4lSs44FgJkZkJ-LLE z%V8#?d8QN75ecknH5xN z6Nf_u-dSkQP}4m7$?=4$m}4)TS?+7}DpRv-%-3>~88d-*5G4MbH})7b>z&m6NB)pw zj^!AoUu7%({RRJc;>$2anoCN^=R8?-{o#=V^Tfp#M3pLy?BJ@kc(Hji{T=8wXE_pd zD(%B%IH_$T@%;=m^X@hgad5qQ1gdJBfaOu|$d2RDX$TH>EMM$R>i%A6b5EqsF#Arn ze6N3HRGp?Os03HJOOpwrz&4K3sq-9qgvxO2+MVB)6I|ClKG0cd`mMD;e+`4$&NvO@ z+SxOb=1v6!-`VcIwC|KcAXI%Du&5?uCrs=?)6XjGK`DFDOZFVGPi9I;{S~f-=T-w| zR-2`YW0iqwVLihakmSC*lAkdYRQdCKH1jeq6;-ZH~NLKgD{L<+G!uh5Eqa&u>J;g zKSLUz{TjMIjnJ8wz|bKtErLmon4arhLtGUd*#tQiv)ydS|?aJ5;| zZ23+)C|_Em*;l0gOPbS>^_4^N824{IS*8Ye7XP3xg}MvzO}jT>vlr=0fgGwU+A@6W zHG?TJ-8+}qmgpR*3WjF!SF+%7*@=>Kw2hocu+f=IOgu}?I}$0spMYH#a#*T?UD9BK zl*58ED#8zZH62@j{63y?81YzUu$-SbA^KlT_SjW zJn*3YYbkvG1l2Cb;1orA{`B($Edv1+B0=J~Q^bX=g%9{%gHBPS){Qh!DVU(cxI)Vggt z^+Wx1K67nR$w_*~9Y|c&$&v*+#drx9V_BPWFH~f~*ZVeE(vIzdGgwCC%UHE{W`@w* zYR1;*=XKrp&I(?dSsOtGAdIl+f5UeDav4o$9rx>hTpdXspWqWLS6(|8g$~r36Jh$8 zuT6j~rtr9x(+sSGQHZm(sH~-gf7U~p*hNwqK{2n9gyu2#BoPB`PFi0i z#!TLl`N8}<7FMJMpbfdXTRV=LDmeP~v3E)GG2UBG{HQ~xN;XF6T2CA^oGsio~L@Ad*$k9b-}!;y=o$!^aA2y?a5OTy|WJZ1ijP zgO$99oxCpFtE5C3KRKy@;B$bV>-86g`FJF;1nFZT9Zq6wIG#sTjs|6b7d+r^T8S|{ zVdx~B5%_U~Oe^j`;GNLRyoX^ak^lkT`>IDvTOj1vDxh70*vuqJ#{%PA!4NkzyCb3B z`dmx?@RnRnzxG9eL~y>NVD^)eir~C!hT3B@Ed^&NzdEZL3i)r`T%vwSdC~96uDzw> zOT){Z0niIXs$)~2;_NA(`hiJ#$TrH0KD zIBYXzBv3N_O0~pPN5-YgLRg#b^n@L;(h)ndOxkn+?{yA(+jd;|hAEgT#~COPn@6Ye zYTX@5$9KnkLfX-j_fu9EdG+u%ouxgyxwM8bz;t5UDYyKVW%HeH&6WfS{Rg(dHS*jqz-!t%_}?B;VzTMv^&XY^Z)1J% z{g$?OX`jPpH~%0x_{>ZB@p$y!?1m9yeH8<3|7a9FD%l8RI4K~SebQ$M7VrCD23eXT z5StK&tru+HG4Iv6a^kVh6SZ{x-IDKNq;$WK%JG)=(@r5uyOKsBKJ?|p;*0j}Rh&Y? za6^#!e(!-_zxjGRjvN|iT+$>JP{}3?=SiF`&YO3{Bg-3MVY^cgszOo|a)mTyMe8|m zF+UI3!$QY-mL0^b+Q8y*G*^mvUUmj?kf-Z1?0YEn{{Jol_I6qX`oA>=rvE;uA^g8C z!qvpl$->Uo*6u$ik)oz$x5fH@_7wD!dBkH#$~4~U^CjYGG4!XUqT9#8hy(bgnrCDP zK_gqwR)6lex-Sr?L@e=1J#4x64m#M!b{R5dZGU4&nMtY%7s!dE7dc1{v3)Xkh0PQ) zk5okFUv^?hzpWwi-?8Z00%=nbK`p{(W%vFV`Sa4c`%ylL^la9I@uwM+9Z5lRYKYNR za#vx@*yrYzYfqgjMkuDvGfkpjTOy+SsPFw@sap@OAkRs58V-2K)CLMSKQ_ z0VOBA!1Zv9zm1;FGX-$#2i~>yR3h?yR~Aqeo*|KBKOvu`$+3SPChn!zP{;$j2;dOW z++v1+i+msIOEz{2DQPX1PgcwgsL)dlcWT;Ir>UNTq7 z#joK6imxE_-h8Ye(C!eGdzMG+anDVSV>7HhBSw+z&lm0xFiWg4k6nJvC{#Nt3gFp& zRx0}S=c^SUXa$8_FI+BhBu8Zz)S;?){}i-{RJd6=4alXF|>@YYAFWljoigv);BNG;UZQI#Q)&k7!ee#?uTX zCZ~Kx${ySvG(i2ARVLUFKVNqD^558xaP0Bhv1OK6X*n1tt`lD~;E7*Jr&8uj&)ASbtg})-S)hu*Eso>)_ti9Ye3b=oS(G`REg)IJ z>Fc?`oNZ!n&dxw9HON?$FLs@un}k*#juu!q*}sPE2e4x9?wbRd8C34@1{=*P5Y!bY z8R^f6VctM5Og7HfI2m~}=KBp-R-KAO@(+-oSW+eZwX>b_06ZxXRtQQKZaF|#eM?D3 zE!R@^sI2tu%R(KkeCacn`K5%YnD$Z5$1Ik@X);+B^%&n=)rylc`!o`%=oCKPfs;OK zyY{*|Xms41TeEVvp$V0h$mJd7TRq}^pZ}&<0iq_4*v%AnM0~34UQ6$bNI$*K(0)AP zQ{tr?`@H0-IJ4H@N)}ja>1)S@wYu$e{i@3J#sgCMq}%}WZu*~1M+rm>Tj<_ZjM9n~ zwRpx+GKg<6y?`WJ^04A&W~FOR69qE3cFJh;B>A@-LNJ(gr4*?OiqWn4PB1eHOxjmgy3)Wwh*hIhXjRI2`lKnoVp6U`|d}Dw<9vOZh(s7`p2l7vSc#XTdM2 z(xr(%@c#ucUHi30z5mE-hwA?WU~c9nCf5JSW{;+x(-teD-&z^}#YIYY5qX)6BaU23 z1=3bcnQV5`&eKc@1{fg$1dNE~Kl;|g%dS1z-*^y-4F_CVI95=7w63U~z1zS|teb?X zHnBg9yocbEz!*8DR4LxGYHoZwy0}rk=%i@Xz=2M+5l^MZ5k*8k2=Q54y~HK%h<`Im ziO3z)AJvH>6rNIjdw!49#A;K9BA_2MRjhgH9KtV&kg17;WKtAshx&{f_8rxx3X0XF zS2LNirxlD#hg1oR|+Qo#Z8k6SgL!X0FB(BaJU9Xs;Ahs8G3d*K307 zWCRu1Gyh#}ioftwHXN;HLVBJ((sU-Lk&8^R6t${+2*kzmc~z;{wxNi@J=qL~VnBUS$TF%(djV}O zXHKe5%`QKbdsgvRXCNfVVj?a{TE$4_I5A#*m_ghuo%)TWgp0rt%$^sxCO5&d*t@Jp z#67*9vRRjMYkuTwW4%1j_;j4$@8#TvXdif~$@Fls(06n~Xx86~?7(H>xJRm@K&&O-{DVyAJ)x zJyDq)QGwU2x(f@f8ET2=CaI-FOrj^{1{TMrn`S+0ml1A)0L9KZL}t6inu^mYSUwu3<~dAK5!K&_f_rp|lGz>oa2vw&a7q+pz4?7Oc> zBwYwh0e#^Rwb~qPn5HQT!38e`RWu>_%ddv9^jMbo=hflo{k4})NTiH=)y$#K$ zx>-TwbzWV;GtS$-)Izrb?#5Zflq$nxtmBlqUP7Y|<$7`0d6Gu10(G4j8n?PVuA3KU z=oB|Co6aq%81`5Ny7F$?zD;yyKW|tLEz7*@E&C|wO8D9D0&WKD-upy&=OgC7TR00a zeBMP1bG%z9jWTwt3`0b}!F|R5XOIot9BtA25E{-))q~)}JeP-}JQP6})f{^6rKFS? zp`*Ve$nt%iQ1w*w1T4OS#b^{y)q!uQp{x<97lm!fUg?BnJU5aSSS(ZZ?xDg+c~uR4 zD?)5eO2wf=8a?8f(^kB#fH|+slcx4ysNLu;K5o zLmSgy>I~54*l-9Xh5xPIpNR?JY18{<({b61k#tCcx6o*bP3<>B!|oyTebj8CFbysAUQ>z9j`eYUdl;Vx>AM1Eb-| zg9)C6i-%Abf`NWsiDj9nteZ@g`7oERGhKYAOojj(VmpE;@J5NB8&vx?+tZaf786Flnjbf>B$!tLwISA?UiFTL}eItS_}q+lQ4M31$)VYm%b%^a&iI#U6h^bmCPr zZxamO)!Lgc4#}C>0N^7Kb5n~5%s5c0ByOi4X8wZk`{`MduA72_d4(FI>#eM(6?j3t z56NO-WsFlo&~e~s(Pgi<6gJ%TAjpg!9Nq7f*ZTGDFTWsBV`~Ah62v3p2E-Q`Uuv;E zj67T#IN2W~W*@}+QL8HO<_-?R{^V^y*{=U~YZgU8h^O{fF5`S2d@M2Vgg{O2BilLEydnxn> z+cNpd3nt#49?qT$c1NA-W=nSBs)A^kL`08k$gJ*-aoc)&v2n9>5xp05`DZz1&I=DA z&tGP9UF%Ty32lhcYO~D!k&F;rC(6aEm{0sP zrK=@^os_%O3uW~{*Fz`OnMUQQ62Uwbf5v}bW&iQ;7%zIJF!a@zG%`{C-+C4Ur`QT+ z&kFb@uUY&Wes!j;$Bh_RyMIIe(QBiF$n!CdZ@5$J44^9-yitAcTkK@p;{}N?JOFwn zL|kOB6J5BZ3i;D@;mxzzoa`RQou0t+^~(we@~iniG~Z z%tH)r)-4)*3JqN-KG2Rkf=*!Z$KiALi9FV9K3pJ5j0V>5Tdj<*vReiK$et#=BK&1b zRMx3{Er7`kD9nMp8A5gHBG~x9e8#~DZAF*~2^B-BLREjBNK`s?B9_(y^w(6$-FuL_ zKIE2uL119$z%F8lT|SjYb%hC}Op`3sCeU7;P2?0pqc-M*OmspdV;jgKaEB$@?`f=L zI~+Tiz1B$sQb=zxa(ER$9^dV~J+<1a6l(>O=4Db6-u#7$kq5fBfE-@4M>QBd+5u=_ z)L1OaKRM>AeAlRGHm{WII6rHSOWE-Z(7!l%3wucq7FB@qooWjV6vIxLBQ+6>W1V>l z^A&-dRZBBIW21bhGQ4(=;-|%upt?7eY1x;slEdOROZ9?RfNIRMW$rCyU$xx$=76V! z$Mdx@AwhJJc7rR-sh;5vPrysHJt4;Sms+n%QBpE9{tqY=qg4($eeToBbpd;F zsBz;FZ62#}fr8P>THpdp+sjvw?4rNP3)?H_o^*fv7*T2O%C8ihx*F%ctr}M@K{`k! zcVjHAz13AK$wYY))9t#XS6;0joKE4|$!rvu zH~!r`@zwB~T@w~Dv9zqrRS&$*Ce=dkr*%JC; z0(r#*_LS_d2ky3j^V55GXt|`C&CuY{%%`e1iVgo&1_`3!_o{meN!pTNu6UIxXFiJ3jX(( zn9zw1;f)LP>gL6d{ilh=FC-u#AR_QbKaAhW%^oIb=jQF{XWK0yBkiG79Gsi2UAR=CM?DE-E!>m;Z%T!1je-$Mn0E+=kwj zSCYRg-)l7PIWA5S_|XvIua++SMsj^8r_;@CO>-=7>x0^@OwEr3YN`rkxO@N;>-WtL zIOZ-Y@JxvQt!~wC)@1>EKUxO`?I@CnZXPRwOWql85+2GNn|<+YO&KE)FP=ud|HM+7 zP+c6cN^?RBnd|XtZ7O8o_bWA+l@*9Ete=Ix_3l_TAcSf$=wwAF?VPOJ^XD(L-JLM( zi7YuBM1$*vdFjAtqh=i|*YTFo8QiyXRo=SzYWA)=+#id=NEt>3^O-dGRCS6Y!}ex~ zp(#15wVgV1&J_kiCOmNMZw6_)HA$b*voxIbnj;F!mt@U(i_(?14Sl}bvd6qQv9kHd z5*sr?^qPLwQSQf1H9%Hhwc1Rh(ok)DZ`TiT(p2^uH4`-n7 zM5~muhxfy{U{scx1$Y=gku$*gb{vN-tezl=)RSRh{PST;mh<-}6P{(76A=1t`AqD% zx|*JPv(cu*wu19G;hbdx(?@1>4l=SIl^K0i-P8Ei-Ma}$>xBEnPHED&-;AtPrpk-Z?k~n zeM4E8w~g&Sq1`H|BtiNwi(5*oC8@y^o;C{#Ng=SRMi0d}8PS{XO6Ue>RFV%yjAlc+ z;jaLApHw1&{Z9yM#8B!19u=hBE8Eb%U{&ysBe_yr^(P}emtLhK> zq8|-GBC4R%<``7*ikB0wV^c!cf#NEj+7AKOHJ48kxS7Gmw&*LyQ6m~M$A06E`h^d} zu8QCjIT*2WP~A7l?kI5JTz0Or$9F8N&lL9cRkqB`+yXc&^pD1GbfrbZmuk(-iQ}#o z0^haK3Va=Iq!o@x&4JmzNoEp4Xm(Tj)Nr;KjM40nmR(~Jyb*OHtHoDN?%teuU`JZ1 zd%GX#e3Ij2wXfZ5pLz~#Jb3y!2i3H!P?X^PD!P;*e!G1)8m4GW@xqhaHt`#ihs_7L zHDlf5af3UAFEho=5}sxyD75%Disv6%N_l=JahlV;q*Xwc3)t#1f`@58>q>bDRS6}? z7mMnKQmQ<;VftQKn$CE~UDJc{0X)6y4u{6-71}Q00b&OmzD-G_;EnYD6KZxm7veKT!iq?Pmc z9ka7MXCJ?*?q@OYrs{h3I8}ECVY)KdMG-!)Zvh&m=P*Sg%$`=D_{rIr&+H&+48)IF zix?VOB|m(9me`F(H~7$cu|kiY$(|%W?jsc)V+>13Qv{#BRH#&TOta|(fu-zRQdM}D zDIN{yYTQB~=vl?(B?l^Xzbmm-%|e9Czd@)M$m)Gv%LDLVs99VKVD2AdenctIZ)I@5 zL9JI=n9HJC##|{IVq1>Ez}}yRw3dWu6d5PW#PHL^cs!yXMx<_8rGJvwj*TjxTE{0L zY}^1W!S%{EI$B5*(PIjdSHZKoPsqKtA;#CotSnoDVpJpY>yLA9Fk#=A7R6+EuAe;Tle* z027qzg&yOqRUmASm<+*6DsoUU3N#KuzR*9YDh#fo40Z&q8FjnH zpPYoaiw)l%V!P$dNnKMIFLJjW`2bhXS_B0W<6K_(IM3?f*p%FP`1O(L4;SRQ3wq!? zHz=>!?uUGzMGeP1ND15r5^fY7=em-6v#pm?-8@I#`I+I4x|i{W>P1hW<#uiI0LJ6N z$;;_LX1hp84pNidG$MSqR3#Um{N6H$;n}M6h^j_`uFewzB`1>=rcgDpuR|7XhHg)< zD)h6>7(A)TvFDj*DB}H;nL@|R22S$_%;$w zG7Y}?nGAz=w1=8s`@OTKaNH;P^K)r8MNR_&Jg0(Kk4{tPox+Rw6a!KKfbn;aPEECo zm=8VZntrR>tT;szhL<2<3=lQmo0Bqmg?~U?$JkSi=UHr&QVFq<^>rOJ%s$hgr)y2L zK>1$Th2;jiwb7f=s^OA74aT^xezWnxD?0NcX)NNCtix z(U%c%JK+0`CpEN_r(B$6u61wa>#Zt0ZhWn*@)O?GUhmuH2=CJ0ewdN2UenY(U*(pG zrke~5d86sJ;r7-Hv_tQh93G9Dy>vri^wl)2NHzI&nf}WDapYU?T0aq2bW>}m?`@XW zw;uOO%?_%L)eJDfQfGuUMy_eOdgyuA++;=ynf$GH7I5Q&IBjbU~^BuC>z2 zDK-{(V{Qn)k9*frf>0o-NZ4(O$$KoRYk-z7`$u#zm}}G|Qbg&%-oLzEy}5nn4JKrNG?<<3lEzpirMhh~D<_~3Z zKRDHy7D|lsRbU!FgoJ2j#LZibmk(q`%d#xHtrd4Z9$^j({n5F*c#KYjZRVXmxgJbf zU$kX#Y$taU?nA#F^viT&RinTPxs-_ng8Gs((=V?;wvJ2elH!=H=U54jR$h<_CcG# zaQpj~vAnH$;idqKV%_H*rS*uk8BOJu3Q2ydtX|d@+_-rX-R3!x_PflN`i0?)L)Wug z)@NX4Gw9z zbqJ@UNdf8o3g*{jYJ*Na5Zxt_@1Y$%HC~X80>*6();L}ouVBkqvJ9j(6I_h_IJf2c zqwsKn<(F#PTZHl7G-nh_^A}^lUDiC*h7s06C=44Be_HLLaDs(e`X_|J4dZa4C<05q z5Z&&QGYCH$FlS3s_(oLHjGu2xH#5)tBCiejc90=qkY#F`^lj%~H#Uf{h4y z?iKd90;6|#+9`mS?s7^ibS`jbSMb@NiS6q%*V;-*@(Kh3l6OjhSB0bd%d;uV$5`sv zBH{2hwh+`~j{S)t<}i=F_}Bm{wrJYiiP{}QcRSC89(N=!UQqGEXC!@4P$wg2fWWBG zG9G;BoK|*uI8t}I!sTI&-8sx}qY^?DS8VNPTM4P1*zTH6CQLD5230_6`?tFs{oJWP zvF6UVLX-EgQddT&allTv06@*f|NLS!?$5cn|9lMSLKU@im3#)e|)C zqRgJ|2pxAT!d->3Xm!?pqJy0`A5um16KD@sEXwagWy28HFOi3KyV{QFb)mvpI(Oqd zg1#s^j82?b)G$TQ$Du54ndC zi#bB?B8K5MhbjoBrkA^T?HWfrdG`f9_t3}rDNH5)dIuWe=DXyvX zfp#g0n7YP;QCb+KbIU#f1yc1qe%XA}mHHOrM+|0~jFiK%O)JR9hBRJ3n;Y3nH&Gbc z@2fy~G=r*dcw^UmDOp>`p=c+g99)8szE{;ZNL9<&E*4p3SEqeV@^os?kPc@4O*8&F zuuRrvxRx?={7xeYVvp2jYd)!bopFS?Y`r<%%wKt~nXu-oaTlq>)_wvzuT(R|r+pXk zhMJF!=$i7@RR<(Krw6@JJaYFjYsm6{wLuGo5kJ{ z`-{b)`-lCxKNg$Co_77kT5e_IOwk-~bZ`X!a5 r@f)e1uUObh*rofIlAqRZl>RE@+8UQG?%hAy@Z Date: Fri, 26 Apr 2024 19:05:57 +0200 Subject: [PATCH 09/28] No longer include license data #3459 These are now provided by the separate licensecode-data and licensecode-index packages. Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-mini.toml | 8 +++++++- pyproject.toml | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pyproject-mini.toml b/pyproject-mini.toml index 770d3cb242e..e428d55139b 100644 --- a/pyproject-mini.toml +++ b/pyproject-mini.toml @@ -124,6 +124,7 @@ packages = [ ] testing = [ + "flot", "pytest >= 6, != 7.0.0", "pytest-xdist >= 2", "aboutcode-toolkit >= 7.0.2", @@ -168,6 +169,9 @@ includes = [ ] excludes = [ + # these are now provided by separate, smaller packages + "src/licensedcode/data/**/*", + # Python compiled files "**/*.py[cod]", "**/*.egg-info", @@ -329,7 +333,9 @@ norecursedirs = [ "lib", "lib64", "Lib64", - "Scripts", + "Scripts", # these are now provided by separate, smaller packages + "src/licensedcode/data/**/*", + "thirdparty", "tmp", "venv", diff --git a/pyproject.toml b/pyproject.toml index 3feb3886972..184eb7d50ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,7 @@ packages = [ ] testing = [ + "flot", "pytest >= 6, != 7.0.0", "pytest-xdist >= 2", "aboutcode-toolkit >= 7.0.2", @@ -167,6 +168,9 @@ includes = [ ] excludes = [ + # these are now provided by separate, smaller packages + "src/licensedcode/data/**/*", + # Python compiled files "**/*.py[cod]", "**/*.egg-info", From 042d2d5693a4908ac518a157784dd6cebd8bfbc2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 19:07:07 +0200 Subject: [PATCH 10/28] Adjust configure script to use flot #3459 We must install flot first before doing any other installations. Also cleanup dead code. Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- configure | 21 ++++++++------------- configure.bat | 20 ++++++++++++++------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/configure b/configure index 9d496b9f653..23d36a94323 100755 --- a/configure +++ b/configure @@ -149,11 +149,12 @@ if [[ $OSTYPE == 'darwin'* ]]; then fi # Requirement arguments passed to pip and used by default or with --dev. -REQUIREMENTS="--editable $BASE --constraint requirements.txt --constraint requirements-linux.txt" -DEV_REQUIREMENTS="--editable $BASE_DEV --constraint requirements.txt --constraint requirements-linux.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" -REL_REQUIREMENTS="--requirement etc/scripts/requirements.txt" -PROD_REQUIREMENTS="scancode_toolkit*.whl" +REQUIREMENTS="--no-build-isolation --editable $BASE --constraint requirements.txt --constraint requirements-linux.txt" +DEV_REQUIREMENTS="--no-build-isolation --editable $BASE_DEV --constraint requirements.txt --constraint requirements-linux.txt --constraint requirements-dev.txt" +DOCS_REQUIREMENTS="--no-build-isolation --editable .[docs] --constraint requirements.txt" +REL_REQUIREMENTS="--no-build-isolation --requirement etc/scripts/requirements.txt" +PROD_REQUIREMENTS="--no-build-isolation scancode_toolkit*.whl" +FLOT_REQUIREMENTS="etc/thirdparty/flot*.whl" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -175,16 +176,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin -################################ -# Install with or without and index. With "--no-index" this is using only local wheels -# This is an offline mode with no index and no network operations -# NO_INDEX="--no-index " -NO_INDEX="" - - ################################ # Thirdparty package locations and index handling # Find packages from the local thirdparty directory if present +# offline mode for scancode installation with no Pypi index at all thirddir=$CFG_ROOT_DIR/thirdparty if [[ "$(echo $thirddir/*.whl)x" != "$thirddir/*.whlx" ]]; then PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty" @@ -257,7 +252,6 @@ install_packages() { "$CFG_BIN_DIR/pip" install \ --upgrade \ - --no-build-isolation \ $CFG_QUIET \ $PIP_EXTRA_ARGS \ $1 @@ -320,6 +314,7 @@ PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" find_python create_virtualenv "$VIRTUALENV_DIR" +install_packages "$FLOT_REQUIREMENTS" install_packages "$CFG_REQUIREMENTS" . "$CFG_BIN_DIR/activate" diff --git a/configure.bat b/configure.bat index fd93d5ea423..9635657abfb 100644 --- a/configure.bat +++ b/configure.bat @@ -26,10 +26,13 @@ @rem ################################ @rem # Requirement arguments passed to pip and used by default or with --dev. -set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" -set "REL_REQUIREMENTS=--requirement etc/scripts/requirements.txt" +set "REQUIREMENTS=--no-build-isolation --editable . --constraint requirements.txt" +set "DEV_REQUIREMENTS=--no-build-isolation --editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DOCS_REQUIREMENTS=--no-build-isolation --editable .[docs] --constraint requirements.txt" +set "REL_REQUIREMENTS=--no-build-isolation --requirement etc/scripts/requirements.txt" +set "PROD_REQUIREMENTS=--no-build-isolation scancode_toolkit*.whl" +set "FLOT_REQUIREMENTS=etc/thirdparty/flot*.whl" + @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -54,7 +57,7 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling @rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -@rem # offline mode for scancode installation with no index at all +@rem # offline mode for scancode installation with no Pypi index at all if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--no-index --find-links "%CFG_ROOT_DIR%\thirdparty" for %%i in (scancode_toolkit*.whl) do set "CFG_REQUIREMENTS=%CFG_ROOT_DIR%\%%i[full]" @@ -164,7 +167,12 @@ if %ERRORLEVEL% neq 0 ( "%CFG_BIN_DIR%\pip" install ^ --upgrade ^ - --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %FLOT_REQUIREMENTS% + +"%CFG_BIN_DIR%\pip" install ^ + --upgrade ^ %CFG_QUIET% ^ %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% From 520685eac6fe73ef6723428ea33027e9aaba9fdf Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 19:52:38 +0200 Subject: [PATCH 11/28] Do not include tests in builds #3459 Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-mini.toml | 15 +++++++++------ pyproject.toml | 11 ++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pyproject-mini.toml b/pyproject-mini.toml index e428d55139b..e3934c28167 100644 --- a/pyproject-mini.toml +++ b/pyproject-mini.toml @@ -172,6 +172,11 @@ excludes = [ # these are now provided by separate, smaller packages "src/licensedcode/data/**/*", + # scancode_toolkit.egg-info is only for editable local config + "src/scancode_toolkit.egg-info/**/*", + + "docs/build/**/*", + # Python compiled files "**/*.py[cod]", "**/*.egg-info", @@ -196,10 +201,10 @@ editable_paths = [ "src", "tests" ] sdist_extra_includes = [ "etc/**/*", - "docs/**/*", + # "docs/**/*", ".github/**/*", - "samples/**/*", - "tests/**/*", + # "samples/**/*", + # "tests/**/*", ".gitattributes", ".gitignore", "*.LICENSE", @@ -333,9 +338,7 @@ norecursedirs = [ "lib", "lib64", "Lib64", - "Scripts", # these are now provided by separate, smaller packages - "src/licensedcode/data/**/*", - + "Scripts", "thirdparty", "tmp", "venv", diff --git a/pyproject.toml b/pyproject.toml index 184eb7d50ee..740800542ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,11 @@ excludes = [ # these are now provided by separate, smaller packages "src/licensedcode/data/**/*", + # scancode_toolkit.egg-info is only for editable local config + "src/scancode_toolkit.egg-info/**/*", + + "docs/build/**/*", + # Python compiled files "**/*.py[cod]", "**/*.egg-info", @@ -195,10 +200,10 @@ editable_paths = [ "src", "tests" ] sdist_extra_includes = [ "etc/**/*", - "docs/**/*", + # "docs/**/*", ".github/**/*", - "samples/**/*", - "tests/**/*", + # "samples/**/*", + #"tests/**/*", ".gitattributes", ".gitignore", "*.LICENSE", From 4860e9204e5137db92fbbc39746054444db1bf83 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Apr 2024 19:53:05 +0200 Subject: [PATCH 12/28] Adjust release scripts for flot #3459 Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- .github/workflows/scancode-release.yml | 94 ++++++++++--------- .../scancode-create-pypi-wheel-and-sdist.sh | 22 ++--- .../scancode-create-release-app-linux.sh | 5 +- .../scancode-create-release-app-macos.sh | 7 +- .../scancode-create-release-app-sources.sh | 8 +- .../scancode-create-release-app-windows.sh | 5 +- 6 files changed, 70 insertions(+), 71 deletions(-) diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml index b80549d8944..8b4915b0449 100644 --- a/.github/workflows/scancode-release.yml +++ b/.github/workflows/scancode-release.yml @@ -20,11 +20,11 @@ on: permissions: {} jobs: - build_scancode_wheel_for_pypi: + build_scancode_wheels_for_pypi: permissions: contents: read # to fetch code (actions/checkout) - name: Build PyPI wheels + name: Build PyPI packages runs-on: ubuntu-22.04 defaults: @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: true matrix: - pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"] + pyver: ["3.8"] steps: - uses: actions/checkout@v4 @@ -44,46 +44,14 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Install requirements then build main and mini wheel - run: etc/release/scancode-create-pypi-wheel.sh - - - name: Collect built wheels for ${{ matrix.pyver }} - uses: actions/upload-artifact@v3 - with: - name: wheels-${{ matrix.pyver }} - path: dist/*.whl - - - build_scancode_sdist_for_pypi: - permissions: - contents: read # to fetch code (actions/checkout) - - name: Build PyPI sdist archives - runs-on: ubuntu-22.04 - - defaults: - run: - shell: bash - - strategy: - fail-fast: true - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" + - name: Install requirements then build all PyPI packages + run: etc/release/scancode-create-pypi-wheel-and-sdist.sh - - name: Install requirements then build main and mini sdist - run: etc/release/scancode-create-pypi-sdist.sh - - - name: Collect built sdist + - name: Collect built wheels and dist uses: actions/upload-artifact@v3 with: - name: sdists - path: dist/*.tar.gz + name: wheels + path: dist/* build_scancode_for_release_linux: @@ -93,6 +61,9 @@ jobs: name: Build app Release for linux runs-on: ubuntu-22.04 + needs: + - build_scancode_wheels_for_pypi + defaults: run: shell: bash @@ -110,6 +81,12 @@ jobs: with: python-version: ${{ matrix.pyver }} + - name: Download wheels artifact + uses: actions/download-artifact@v3 + with: + name: wheels + path: dist + - name: Build linux app archive run: etc/release/scancode-create-release-app-linux.sh ${{ matrix.pyver }} @@ -127,6 +104,9 @@ jobs: name: Build app Release for mac runs-on: ubuntu-22.04 + needs: + - build_scancode_wheels_for_pypi + defaults: run: shell: bash @@ -137,6 +117,7 @@ jobs: pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: + - uses: actions/checkout@v4 - name: Set up Python on ${{ matrix.pyver }} @@ -144,6 +125,12 @@ jobs: with: python-version: ${{ matrix.pyver }} + - name: Download wheels artifact + uses: actions/download-artifact@v3 + with: + name: wheels + path: dist + - name: Build mac app archive run: etc/release/scancode-create-release-app-macos.sh ${{ matrix.pyver }} @@ -161,6 +148,9 @@ jobs: name: Build app Release for windows runs-on: ubuntu-22.04 + needs: + - build_scancode_wheels_for_pypi + defaults: run: shell: bash @@ -178,6 +168,12 @@ jobs: with: python-version: ${{ matrix.pyver }} + - name: Download wheels artifact + uses: actions/download-artifact@v3 + with: + name: wheels + path: dist + - name: Build windows app archive run: etc/release/scancode-create-release-app-windows.sh ${{ matrix.pyver }} @@ -195,9 +191,13 @@ jobs: name: Build app source runs-on: ubuntu-22.04 + needs: + - build_scancode_wheels_for_pypi + defaults: run: shell: bash + strategy: fail-fast: true @@ -209,6 +209,12 @@ jobs: with: python-version: "3.10" + - name: Download wheels artifact + uses: actions/download-artifact@v3 + with: + name: wheels + path: dist + - name: Build source archive with deps run: etc/release/scancode-create-release-app-sources.sh @@ -235,7 +241,7 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-22.04, ubuntu-22.04, macos-11, macos-12] + os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: @@ -249,7 +255,7 @@ jobs: - name: Download wheels artifact uses: actions/download-artifact@v3 with: - name: wheels-${{ matrix.pyver }} + name: wheels path: dist - name: test install wheels @@ -258,7 +264,7 @@ jobs: rm -rf venv; \ python -m venv venv venv/bin/python -m pip install --upgrade pip wheel setuptools - for f in `find dist -type f -name "*${{ matrix.pyver }}*"`; \ + for f in `find dist -type f -name "scancode*${{ matrix.pyver }}*"`; \ do \ venv/bin/python -m pip install --force-reinstall "$f[full]" ; \ venv/bin/scancode -clipeu --json-pp - some.file ; \ @@ -295,7 +301,7 @@ jobs: - name: Download wheels artifact uses: actions/download-artifact@v3 with: - name: wheels-${{ matrix.pyver }} + name: wheels path: dist - name: test install wheel diff --git a/etc/release/scancode-create-pypi-wheel-and-sdist.sh b/etc/release/scancode-create-pypi-wheel-and-sdist.sh index 70add72070b..6bea3755860 100755 --- a/etc/release/scancode-create-pypi-wheel-and-sdist.sh +++ b/etc/release/scancode-create-pypi-wheel-and-sdist.sh @@ -9,8 +9,7 @@ # ################################################################################ -# ScanCode release build script for PyPI wheels and sdist. -# Build a wheel for the current Python version +# ScanCode release build script for PyPI ScanCode, license and other wheels. ################################################################################ set -e @@ -18,20 +17,17 @@ set -e #set -x ./configure --dev -venv/bin/scancode-reindex-licenses - -python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" ) -venv/bin/flot setup.py --quiet bdist_wheel --python-tag $python_tag - -rm -rf build .eggs src/scancode_toolkit*.egg-info src/scancode_toolkit_mini*.egg-info -cp setup.cfg setup-main.cfg -cp setup-mini.cfg setup.cfg +venv/bin/scancode-reindex-licenses -venv/bin/python setup.py --quiet bdist_wheel --python-tag $python_tag +# build license data packages +venv/bin/flot --pyproject pyproject-licensedcode-data.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-licensedcode-index.toml --wheel --sdist -cp setup-main.cfg setup.cfg -rm setup-main.cfg +# build code packages +venv/bin/flot --pyproject pyproject.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-mini.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-packagedcode.toml --wheel --sdist venv/bin/twine check dist/* diff --git a/etc/release/scancode-create-release-app-linux.sh b/etc/release/scancode-create-release-app-linux.sh index 94aa847ae47..653786e29d0 100755 --- a/etc/release/scancode-create-release-app-linux.sh +++ b/etc/release/scancode-create-release-app-linux.sh @@ -62,13 +62,12 @@ rm -rf $thirdparty_src_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc -# Build the wheel +# Build the app archive ./configure --dev -venv/bin/scancode-reindex-licenses -venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version cp -r \ dist/scancode_*.whl \ + dist/licensedcode_index_*.whl \ scancode extractcode configure \ *.rst \ samples \ diff --git a/etc/release/scancode-create-release-app-macos.sh b/etc/release/scancode-create-release-app-macos.sh index e689f1f1b04..dd848898008 100755 --- a/etc/release/scancode-create-release-app-macos.sh +++ b/etc/release/scancode-create-release-app-macos.sh @@ -20,7 +20,7 @@ set -e operating_system=macos python_dot_version=$1 python_version=${python_dot_version//.} -python_exe="python3" +python_exe="python$python_dot_version" release_dir=scancode-toolkit-$(git describe --tags) rm -rf $release_dir @@ -60,13 +60,12 @@ rm -rf $thirdparty_src_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc -# Build the wheel +# Build the app archive ./configure --dev -venv/bin/scancode-reindex-licenses -venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version cp -r \ dist/scancode_*.whl \ + dist/licensedcode_index_*.whl \ scancode extractcode configure \ *.rst \ samples \ diff --git a/etc/release/scancode-create-release-app-sources.sh b/etc/release/scancode-create-release-app-sources.sh index 1a73aa7485b..3cbb27dc3e1 100755 --- a/etc/release/scancode-create-release-app-sources.sh +++ b/etc/release/scancode-create-release-app-sources.sh @@ -9,7 +9,7 @@ # ################################################################################ -# ScanCode release build script for a Windows app archive +# ScanCode release build script for a source archive for the apps ################################################################################ set -e @@ -24,9 +24,8 @@ git describe --tags > $release_dir/SCANCODE_VERSION thirdparty_dir=$release_dir/thirdparty mkdir -p $thirdparty_dir -# build an sdist -./configure --dev -venv/bin/python setup.py --quiet sdist +# Collect other built sdist + mv dist/*.tar.gz $release_dir ./configure --rel @@ -51,6 +50,7 @@ cp -r etc/thirdparty $release_dir/etc cp -r \ scancode.bat scancode extractcode extractcode.bat configure configure.bat \ *.rst \ + *.toml \ docs \ samples \ *NOTICE *LICENSE *ABOUT \ diff --git a/etc/release/scancode-create-release-app-windows.sh b/etc/release/scancode-create-release-app-windows.sh index a7957ae15bb..861e0bff40d 100755 --- a/etc/release/scancode-create-release-app-windows.sh +++ b/etc/release/scancode-create-release-app-windows.sh @@ -59,13 +59,12 @@ rm -rf $thirdparty_src_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc -# Build the wheel +# Build the app archive ./configure --dev -venv/bin/scancode-reindex-licenses -venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version cp -r \ dist/scancode_*.whl \ + dist/licensedcode_index_*.whl \ scancode.bat extractcode.bat configure.bat \ *.rst \ samples \ From d5f35690a1ddab6c15d1d8251ceb88094942e270 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 6 May 2024 12:36:50 +0200 Subject: [PATCH 13/28] Use explicit pyproject for ScanCode builds The plain pyproject.toml is for development only Signed-off-by: Philippe Ombredanne --- ...ml => pyproject-scancode-toolkit-mini.toml | 0 pyproject-scancode-toolkit.toml | 360 ++++++++++++++++++ pyproject.toml | 10 +- 3 files changed, 367 insertions(+), 3 deletions(-) rename pyproject-mini.toml => pyproject-scancode-toolkit-mini.toml (100%) create mode 100644 pyproject-scancode-toolkit.toml diff --git a/pyproject-mini.toml b/pyproject-scancode-toolkit-mini.toml similarity index 100% rename from pyproject-mini.toml rename to pyproject-scancode-toolkit-mini.toml diff --git a/pyproject-scancode-toolkit.toml b/pyproject-scancode-toolkit.toml new file mode 100644 index 00000000000..f2f897929e5 --- /dev/null +++ b/pyproject-scancode-toolkit.toml @@ -0,0 +1,360 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "scancode-toolkit" +version = "32.1.0" +description = "ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts." +readme = "README.rst" +license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +requires-python = ">=3.8" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] +keywords = [ + "open source", + "scan", + "license", + "package", + "dependency", + "copyright", + "filetype", + "author", + "extract", + "licensing", + "sca", + "SBOM", + "spdx", + "cyclonedx", + "package-url", + "purl", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "licensedcode-index", + "attrs >= 18.1,!=20.1.0;python_version<'3.11'", + "attrs >= 22.1.0;python_version>='3.11'", + "Beautifulsoup4 >= 4.0.0", + "boolean.py >= 4.0", + "chardet >= 3.0.0", + "click >= 6.7, !=7.0", + "colorama >= 0.3.9", + "commoncode >= 31.0.3", + "container-inspector >= 31.0.0", + "debian-inspector >= 31.1.0", + "dparse2 >= 0.7.0", + "fasteners", + "fingerprints >= 0.6.0", + "ftfy >= 6.0.0", + "gemfileparser2 >= 0.9.0", + "html5lib", + "importlib_metadata", + "intbitset >= 3.0.2", + "jaraco.functools", + "javaproperties >= 0.5", + "jinja2 >= 2.7.0", + "jsonstreams >= 0.5.0", + "license_expression >= 30.1.1", + "lxml >= 4.9.2", + "MarkupSafe >= 2.1.2", + "packageurl_python >= 0.9.0", + "packvers >= 21.0.0", + # use temp advanced patched release + "parameter-expansion-patched >= 0.3.1", + "pdfminer.six >= 20200101", + "pefile >= 2020.1.1", + "pip-requirements-parser >= 32.0.1", + "pkginfo2 >= 30.0.0", + "pluggy >= 1.0.0", + "plugincode >= 32.0.0", + "publicsuffix2", + "pyahocorasick >= 2.0.0", + "pygmars >= 0.7.0", + "pygments", + "pymaven_patch >= 0.2.8", + "requests >= 2.7.0", + "saneyaml >= 0.6.0", + "spdx_tools == 0.8.2", + "text_unidecode >= 1.0", + "toml >= 0.10.0", + "urlpy", + "xmltodict >= 0.11.0", + "zipp >= 3.0.0; python_version < \"3.9\"", + "typecode >= 30.0.1", + "typecode[full] >= 30.0.1", + "extractcode[full] >= 31.0.0", +] + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + + +[project.optional-dependencies] +licenses = [ + "licensecode-data", +] + +# no impact but added for symmetry with the other pyproject.toml +full = [ + "extractcode[full] >= 31.0.0", + "typecode[full] >= 30.0.0", +] + +# linux-only package handling +packages = [ + "rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux'", + "regipy >= 3.1.0; platform_system == 'Linux'", + "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", +] + +testing = [ + "flot", + "pytest >= 6, != 7.0.0", + "pytest-xdist >= 2", + "aboutcode-toolkit >= 7.0.2", + "pycodestyle >= 2.8.0", + "twine", + "black", + "isort", + "vendorize >= 0.3.0", + "pytest-rerunfailures", + "bump-my-version", +] + +docs = [ + "Sphinx == 5.1.0", + "sphinx_rtd_theme >= 0.5.1", + "doc8 >= 0.8.1", + "sphinx-reredirects >= 0.1.2", + "sphinx-autobuild", + "sphinx-rtd-dark-mode>=1.3.0", + "sphinx-copybutton", +] + + +[tool.bumpversion] +current_version = "32.1.0" +allow_dirty = true + +files = [ + { filename = "src/scancode_config.py" }, + { filename = "pyproject.toml" }, + { filename = "pyproject-scancode-toolkit-mini.toml" }, + { filename = "pyproject-packagedcode.toml" }, + +] + + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/**/*", +] + +excludes = [ + # these are now provided by separate, smaller packages + "src/licensedcode/data/**/*", + + # scancode_toolkit.egg-info is only for editable local config + "src/scancode_toolkit.egg-info/**/*", + + "docs/build/**/*", + + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src", "tests" ] + +sdist_extra_includes = [ + "etc/**/*", + # "docs/**/*", + ".github/**/*", + # "samples/**/*", + #"tests/**/*", + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.ABOUT", + "*.toml", + "*.yml", + "*.rst", + "*.py", + + "requirements*", + + "configure*", + "extractcode*", + "scancode*", + + ".dockerignore", + "Dockerfile*", + # path to executable used by a built app + "PYTHON_EXECUTABLE*", + ".VERSION", +] + + +[tool.isort] +force_single_line = "True" +line_length = 88 +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +skip = "doc,venv,tmp,thirdparty,build,dist" + + +[project.scripts] +scancode = "scancode.cli:scancode" +scancode-reindex-licenses = "licensedcode.reindex:reindex_licenses" +scancode-license-data = "licensedcode.license_db:dump_scancode_license_data" +regen-package-docs = "packagedcode.regen_package_docs:regen_package_docs" + + +# These are configurations for ScanCode plugins as entry points. +# Each plugin entry hast this form: +# plugin-name = f"ully.qualified.module:PluginClass" +# where plugin-name must be a unique arbitrary name for this entrypoint. + +# scancode_pre_scan is the entry point for pre_scan plugins executed before the +# scans. See also plugincode.pre_scan module for details and doc. +[project.entry-points.scancode_pre_scan] +ignore = "scancode.plugin_ignore:ProcessIgnore" +facet = "summarycode.facet:AddFacet" + + +# scancode_scan is the entry point for scan plugins that run a scan after the +# pre_scan plugins and before the post_scan plugins. See also plugincode.scan +# module for details and doc. +[project.entry-points.scancode_scan] +info = "scancode.plugin_info:InfoScanner" +licenses = "licensedcode.plugin_license:LicenseScanner" +copyrights = "cluecode.plugin_copyright:CopyrightScanner" +packages = "packagedcode.plugin_package:PackageScanner" +emails = "cluecode.plugin_email:EmailScanner" +urls = "cluecode.plugin_url:UrlScanner" +generated = "summarycode.generated:GeneratedCodeDetector" + + +# scancode_post_scan is the entry point for post_scan plugins executed after the +# scan plugins and before the output plugins. See also plugincode.post_scan +# module for details and doc. +[project.entry-points.scancode_post_scan] +summary = "summarycode.summarizer:ScanSummary" +tallies = "summarycode.tallies:Tallies" +tallies-with-details = "summarycode.tallies:TalliesWithDetails" +tallies-key-files = "summarycode.tallies:KeyFilesTallies" +tallies-by-facet = "summarycode.tallies:FacetTallies" +license-clarity-score = "summarycode.score:LicenseClarityScore" +license-policy = "licensedcode.plugin_license_policy:LicensePolicy" +mark-source = "scancode.plugin_mark_source:MarkSource" +filter-clues = "cluecode.plugin_filter_clues:RedundantCluesFilter" +consolidate = "summarycode.plugin_consolidate:Consolidator" +license-references = "licensedcode.licenses_reference:LicenseReference" +todo = "summarycode.todo:AmbiguousDetectionsToDoPlugin" +classify = "summarycode.classify_plugin:FileClassifier" + + +# scancode_output_filter is the entry point for filter plugins executed after +# the post-scan plugins and used by the output plugins to exclude/filter certain +# files or directories from the codebase. See also plugincode.post_scan module +# for details and doc. +[project.entry-points.scancode_output_filter] +only-findings = "scancode.plugin_only_findings:OnlyFindings" +ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" + + +# scancode_output is the entry point for output plugins that write a scan output +# in a given format at the end of a scan. See also plugincode._output module for +# details and doc. +[project.entry-points.scancode_output] +html = "formattedcode.output_html:HtmlOutput" +html-app = "formattedcode.output_html:HtmlAppOutput" +json = "formattedcode.output_json:JsonCompactOutput" +json-pp = "formattedcode.output_json:JsonPrettyOutput" +spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" +spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" +csv = "formattedcode.output_csv:CsvOutput" +jsonlines = "formattedcode.output_jsonlines:JsonLinesOutput" +template = "formattedcode.output_html:CustomTemplateOutput" +debian = "formattedcode.output_debian:DebianCopyrightOutput" +yaml = "formattedcode.output_yaml:YamlOutput" +cyclonedx = "formattedcode.output_cyclonedx:CycloneDxJsonOutput" +cyclonedx-xml = "formattedcode.output_cyclonedx:CycloneDxXmlOutput" + + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "bin", + "dist", + "build", + "_build", + "dist", + "etc", + "local", + "ci", + "docs", + "man", + "share", + "samples", + ".cache", + ".settings", + "Include", + "include", + "Lib", + "lib", + "lib64", + "Lib64", + "Scripts", + "thirdparty", + "tmp", + "venv", + "tests/data", + ".eggs", + "src/*/data", + "tests/*/data", + "src/licensedcode/_vendor" +] + +python_files = "*.py" + +python_classes = "Test" +python_functions = "test" + +addopts = [ + "-rfExXw", + "--strict-markers", + "--doctest-modules" +] diff --git a/pyproject.toml b/pyproject.toml index 740800542ff..506f48f903c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ +# note: this is the default pyproject.toml that is used only in development +# we use flot the other pyproject*.toml files to build release wheels and archives + [build-system] requires = [ "flot>=0.7.0" ] build-backend = "flot.buildapi" [project] -name = "scancode-toolkit" +name = "scancode-toolkit-devel" version = "32.1.0" description = "ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts." readme = "README.rst" @@ -46,7 +49,6 @@ classifiers = [ ] dependencies = [ - "licensedcode-index", "attrs >= 18.1,!=20.1.0;python_version<'3.11'", "attrs >= 22.1.0;python_version>='3.11'", "Beautifulsoup4 >= 4.0.0", @@ -105,11 +107,13 @@ Homepage = "https://github.com/nexB/scancode-toolkit" [project.optional-dependencies] +# no impact but added for symmetry with the other pyproject.toml licenses = [ "licensecode-data", + "licensedcode-index", ] -# no impact but added for symmetry with the mini config +# no impact but added for symmetry with the other pyproject.toml full = [ "extractcode[full] >= 31.0.0", "typecode[full] >= 30.0.0", From 5c7bd6475fb115d9c8f3495c507fd87573a6a93b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 6 May 2024 12:38:15 +0200 Subject: [PATCH 14/28] Refine packagedcode packaging - Remove dependencies that are not direct. - Start to abstract some of the dependencies on the licensedcode module. Signed-off-by: Philippe Ombredanne --- pyproject-packagedcode.toml | 31 ++++---- src/packagedcode/alpine.py | 98 +++++++++++++++----------- src/packagedcode/regen_package_docs.py | 7 +- 3 files changed, 72 insertions(+), 64 deletions(-) diff --git a/pyproject-packagedcode.toml b/pyproject-packagedcode.toml index 25f0fd7942f..7673d9cd4ba 100644 --- a/pyproject-packagedcode.toml +++ b/pyproject-packagedcode.toml @@ -49,56 +49,49 @@ classifiers = [ dependencies = [ "attrs >= 18.1,!=20.1.0;python_version<'3.11'", "attrs >= 22.1.0;python_version>='3.11'", - "Beautifulsoup4 >= 4.0.0", "boolean.py >= 4.0", - "chardet >= 3.0.0", "click >= 6.7, !=7.0", - "colorama >= 0.3.9", "commoncode >= 31.0.3", "container-inspector >= 31.0.0", "debian-inspector >= 31.1.0", "dparse2 >= 0.7.0", - "fasteners", - "fingerprints >= 0.6.0", "ftfy >= 6.0.0", "gemfileparser2 >= 0.9.0", - "html5lib", "importlib_metadata", - "intbitset >= 3.0.2", - "jaraco.functools", "javaproperties >= 0.5", "jinja2 >= 2.7.0", - "jsonstreams >= 0.5.0", "license_expression >= 30.1.1", "lxml >= 4.9.2", - "MarkupSafe >= 2.1.2", "packageurl_python >= 0.9.0", "packvers >= 21.0.0", # use temp advanced patched release "parameter-expansion-patched >= 0.3.1", - "pdfminer.six >= 20200101", "pefile >= 2020.1.1", "pip-requirements-parser >= 32.0.1", "pkginfo2 >= 30.0.0", - "pluggy >= 1.0.0", "plugincode >= 32.0.0", - "publicsuffix2", - "pyahocorasick >= 2.0.0", "pygmars >= 0.7.0", "pygments", "pymaven_patch >= 0.2.8", - "requests >= 2.7.0", "saneyaml >= 0.6.0", - "spdx_tools == 0.8.2", - "text_unidecode >= 1.0", "toml >= 0.10.0", - "urlpy", "xmltodict >= 0.11.0", - "zipp >= 3.0.0; python_version < \"3.9\"", "typecode >= 30.0.1", "typecode[full] >= 30.0.1", ] + +[project.optional-dependencies] +full = [ +# we still need all these deps and clenup is needed +#from licensedcode +#from summarycode +#from textcode +#from typecode +] + + + [project.urls] Homepage = "https://github.com/nexB/scancode-toolkit" diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py index 1b155a90c97..6fba791e7aa 100644 --- a/src/packagedcode/alpine.py +++ b/src/packagedcode/alpine.py @@ -17,14 +17,13 @@ from pathlib import Path import attr + from license_expression import LicenseSymbolLike from license_expression import Licensing from packageurl import PackageURL from packagedcode import bashparse from packagedcode import models -from packagedcode.licensing import get_declared_license_expression_spdx -from packagedcode.licensing import get_license_detections_and_expression from packagedcode.utils import combine_expressions from packagedcode.utils import get_ancestor from textcode.analysis import as_unicode @@ -903,8 +902,16 @@ def L_license_handler(value, **kwargs): Return a normalized declared license and a detected license expression. """ original = value - _declared, detected, license_detections = detect_declared_license(value) - detected_spdx = get_declared_license_expression_spdx(declared_license_expression=detected) + + detected = "" + license_detections = [] + detected_spdx = "" + try: + from packagedcode.licensing import get_declared_license_expression_spdx + _declared, detected, license_detections = detect_declared_license(value) + detected_spdx = get_declared_license_expression_spdx(declared_license_expression=detected) + except ImportError: + pass return { 'extracted_license_statement': original, 'declared_license_expression': detected, @@ -1363,53 +1370,60 @@ def source_handler(value, **kwargs): def detect_declared_license(declared): """ - Return a tuple of (cleaned declared license, detected license expression) - strings from a ``declared`` license text. Both can be None. + Return a tuple of (cleaned declared license, detected license expression, license_detections) + strings from a ``declared`` license text. All of the return tuple values can be None. """ # cleaning first to fix syntax quirks and try to get something we can parse cleaned = normalize_and_cleanup_declared_license(declared) if not cleaned: - return None, None - - # then we apply mappings for known non-standard symbols - # the output should be a proper SPDX expression - mapped = apply_expressions_mapping(cleaned) - - # Finally perform SPDX expressions detection: Alpine uses mostly SPDX, but - # with some quirks such as some non standard symbols (in addition to the - # non-standard syntax) - extra_licenses = {} - expression_symbols = get_license_symbols(extra_licenses=extra_licenses) - - license_detections, detected_license_expression = get_license_detections_and_expression( - extracted_license_statement=mapped, - expression_symbols=expression_symbols, - ) + return None, None, None - return cleaned, detected_license_expression, license_detections + try: + from packagedcode.licensing import get_license_detections_and_expression + # then we apply mappings for known non-standard symbols + # the output should be a proper SPDX expression + mapped = apply_expressions_mapping(cleaned) + + # Finally perform SPDX expressions detection: Alpine uses mostly SPDX, but + # with some quirks such as some non standard symbols (in addition to the + # non-standard syntax) + extra_licenses = {} + expression_symbols = get_license_symbols(extra_licenses=extra_licenses) + + license_detections, detected_license_expression = get_license_detections_and_expression( + extracted_license_statement=mapped, + expression_symbols=expression_symbols, + ) + + return cleaned, detected_license_expression, license_detections + except ImportError: + return None, None, None def get_license_symbols(extra_licenses): """ Return a mapping of {lowercased key: LicenseSymbolLike} where LicenseSymbolLike wraps a License object. This is a combo of the SPDX - licenses keys and the Alpine-specific extra symbols. + licenses keys and the Alpine-specific extra symbols. This can be an empty mapping. """ - from licensedcode.cache import get_spdx_symbols - from licensedcode.cache import get_licenses_db - - # make a copy - symbols = dict(get_spdx_symbols()) - ref_licenses = get_licenses_db() - - for alpine_key, license_key in extra_licenses.items(): - # check that there are no dupe keys - assert alpine_key not in symbols - - lic = ref_licenses[license_key] - symbols[alpine_key] = LicenseSymbolLike(lic) - - return symbols + try: + from licensedcode.cache import get_spdx_symbols + from licensedcode.cache import get_licenses_db + + # make a copy + symbols = dict(get_spdx_symbols()) + ref_licenses = get_licenses_db() + + for alpine_key, license_key in extra_licenses.items(): + # check that there are no dupe keys + assert alpine_key not in symbols + + lic = ref_licenses[license_key] + symbols[alpine_key] = LicenseSymbolLike(lic) + return symbols + + except ImportError: + return {} def normalize_and_cleanup_declared_license(declared): @@ -1684,7 +1698,7 @@ def apply_expressions_mapping(expression): """ Return a new license expression string from an ``expression`` string replacing subexpressions using the DECLARED_TO_SPDX_SUBS expression - subsitution table. + subsitution table. Return an empty string if this cannot be mapped. """ licensing = Licensing() @@ -1697,8 +1711,8 @@ def apply_expressions_mapping(expression): try: expression = licensing.parse(expression, simple=True) - except: - raise Exception(expression) + except Exception as e: + raise Exception(expression) from e return str(expression.subs(DECLARED_TO_SPDX_SUBS)) diff --git a/src/packagedcode/regen_package_docs.py b/src/packagedcode/regen_package_docs.py index e6b0110c94e..6ea399b5219 100644 --- a/src/packagedcode/regen_package_docs.py +++ b/src/packagedcode/regen_package_docs.py @@ -16,11 +16,10 @@ from commoncode.cliutils import MISC_GROUP from commoncode.cliutils import PluggableCommandLineOption from packagedcode.plugin_package import get_available_package_parsers -from jinja2 import Environment, FileSystemLoader - TEMPLATES_DIR = os.path.join(dirname(__file__), 'templates') + def write_file(file_path, content): file_path.open("w").write(content) @@ -35,6 +34,9 @@ def regenerate( templates from ``template_dir``. ``test`` is to generate a stable output for testing only """ + from jinja2 import Environment + from jinja2 import FileSystemLoader + environment = Environment( loader=FileSystemLoader(template_dir), autoescape=True, @@ -50,7 +52,6 @@ def regenerate( write_file(file_path=doc_path, content=packages_doc) - @click.command(name='regen-package-docs') @click.option( '--path', From 2b1d2e71e50f5e14b52d3b4fd83d9ee99059a129 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 6 May 2024 23:07:27 +0200 Subject: [PATCH 15/28] Refine packagedcode packaging Signed-off-by: Philippe Ombredanne --- pyproject-packagedcode.toml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pyproject-packagedcode.toml b/pyproject-packagedcode.toml index 7673d9cd4ba..f60306b857e 100644 --- a/pyproject-packagedcode.toml +++ b/pyproject-packagedcode.toml @@ -81,17 +81,6 @@ dependencies = [ ] -[project.optional-dependencies] -full = [ -# we still need all these deps and clenup is needed -#from licensedcode -#from summarycode -#from textcode -#from typecode -] - - - [project.urls] Homepage = "https://github.com/nexB/scancode-toolkit" @@ -104,6 +93,14 @@ packages = [ "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", ] +full = [ +# TODO: we still need all these deps and clenup is needed +#from licensedcode +#from summarycode +#from textcode +#from typecode +] + [tool.flot] wheel_path_prefixes_to_strip = [ "src" ] From 80c72bc847aa4bb8265cee8543121dcf11f3a666 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 00:18:10 +0200 Subject: [PATCH 16/28] Build only modesl in packagedcode Signed-off-by: Philippe Ombredanne --- ....toml => pyproject-packagedcode-models.toml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) rename pyproject-packagedcode.toml => pyproject-packagedcode-models.toml (88%) diff --git a/pyproject-packagedcode.toml b/pyproject-packagedcode-models.toml similarity index 88% rename from pyproject-packagedcode.toml rename to pyproject-packagedcode-models.toml index f60306b857e..cc901764150 100644 --- a/pyproject-packagedcode.toml +++ b/pyproject-packagedcode-models.toml @@ -3,9 +3,9 @@ requires = [ "flot>=0.7.0" ] build-backend = "flot.buildapi" [project] -name = "packagedcode" +name = "packagedcode-models" version = "32.1.0" -description = "PackagedCode is a library to parse and normalize package manifests, lockfiles and dependencies for all type of app and system packages and report them by PURL." +description = "PackagedCode Models is a library with core models of package manifests, lockfiles and dependencies for all type of app and system packages keyed by PURL." license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } # TODO consider a readme? # readme = "README.rst" @@ -16,18 +16,10 @@ authors = [ keywords = [ "open source", "scan", - "license", "package", "dependency", - "copyright", - "filetype", - "author", - "extract", - "licensing", "sca", "SBOM", - "spdx", - "cyclonedx", "package-url", "purl", ] @@ -106,7 +98,7 @@ full = [ wheel_path_prefixes_to_strip = [ "src" ] includes = [ - "src/packagedcode/**/*", + "src/packagedcode/models.py", ] excludes = [ @@ -138,11 +130,7 @@ sdist_extra_includes = [ ".gitignore", "*.LICENSE", "NOTICE", - "*.ABOUT", - "*packagedcode*.toml", - "*.yml", "*.rst", - "*.py", "requirements*", ] From 064c646585d6f38884f76cbbc3cb2c94c282811a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 09:03:19 +0200 Subject: [PATCH 17/28] Move packagedcode.__init__.py code to handlers.py Move packagedcode.__init__.py code to handlers.py to allow distributing multiple smaller packages built with flot. Signed-off-by: Philippe Ombredanne --- src/packagedcode/__init__.py | 261 ---------------------- src/packagedcode/handlers.py | 261 ++++++++++++++++++++++ src/packagedcode/licensing.py | 2 +- src/packagedcode/models.py | 10 +- src/packagedcode/plugin_package.py | 4 +- src/packagedcode/recognize.py | 6 +- src/summarycode/classify_plugin.py | 2 +- tests/packagedcode/packages_test_utils.py | 2 +- tests/packagedcode/test_package_models.py | 2 +- 9 files changed, 275 insertions(+), 275 deletions(-) create mode 100644 src/packagedcode/handlers.py diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index a6d84c1e56e..e69de29bb2d 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -1,261 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import attr - -from commoncode.system import on_linux -from packagedcode import about -from packagedcode import alpine -from packagedcode import bower -from packagedcode import build -from packagedcode import build_gradle -from packagedcode import cargo -from packagedcode import chef -from packagedcode import debian -from packagedcode import debian_copyright -from packagedcode import distro -from packagedcode import conda -from packagedcode import conan -from packagedcode import cocoapods -from packagedcode import cran -from packagedcode import freebsd -from packagedcode import godeps -from packagedcode import golang -from packagedcode import haxe -from packagedcode import maven -from packagedcode import misc -from packagedcode import npm -from packagedcode import nuget -from packagedcode import opam -from packagedcode import phpcomposer -from packagedcode import pubspec -from packagedcode import pypi -from packagedcode import readme -from packagedcode import rpm -from packagedcode import rubygems -from packagedcode import win_pe -from packagedcode import windows - -if on_linux: - from packagedcode import msi - from packagedcode import win_reg - -# Note: the order matters: from the most to the least specific parser. -# a handler classes MUST be added to this list to be active -APPLICATION_PACKAGE_DATAFILE_HANDLERS = [ - about.AboutFileHandler, - - alpine.AlpineApkArchiveHandler, - alpine.AlpineApkbuildHandler, - - bower.BowerJsonHandler, - - build_gradle.BuildGradleHandler, - - build.AutotoolsConfigureHandler, - build.BazelBuildHandler, - build.BuckMetadataBzlHandler, - build.BuckPackageHandler, - - cargo.CargoLockHandler, - cargo.CargoTomlHandler, - - chef.ChefMetadataJsonHandler, - chef.ChefMetadataRbHandler, - - cocoapods.PodspecHandler, - cocoapods.PodspecJsonHandler, - cocoapods.PodfileLockHandler, - cocoapods.PodfileHandler, - - conda.CondaYamlHandler, - conda.CondaMetaYamlHandler, - - conan.ConanFileHandler, - conan.ConanDataHandler, - - cran.CranDescriptionFileHandler, - - debian_copyright.DebianCopyrightFileInPackageHandler, - debian_copyright.StandaloneDebianCopyrightFileHandler, - debian.DebianDscFileHandler, - - debian.DebianControlFileInExtractedDebHandler, - debian.DebianControlFileInSourceHandler, - - debian.DebianDebPackageHandler, - debian.DebianMd5sumFilelistInPackageHandler, - - debian.DebianSourcePackageMetadataTarballHandler, - debian.DebianSourcePackageTarballHandler, - - distro.EtcOsReleaseHandler, - - freebsd.CompactManifestHandler, - - godeps.GodepsHandler, - golang.GoModHandler, - golang.GoSumHandler, - - haxe.HaxelibJsonHandler, - - maven.MavenPomXmlHandler, - maven.MavenPomPropertiesHandler, - maven.JavaJarManifestHandler, - maven.JavaOSGiManifestHandler, - - misc.AndroidAppArchiveHandler, - misc.AndroidLibraryHandler, - misc.AppleDmgHandler, - misc.Axis2MarArchiveHandler , - misc.Axis2MarModuleXmlHandler , - misc.CabArchiveHandler, - misc.ChromeExtensionHandler, - misc.CpanDistIniHandler , - misc.CpanMakefilePlHandler, - misc.CpanManifestHandler, - misc.CpanMetaJsonHandler, - misc.CpanMetaYmlHandler, - misc.InstallShieldPackageHandler, - misc.IosAppIpaHandler, - misc.IsoImageHandler, - misc.IvyXmlHandler, - - misc.JavaEarAppXmlHandler , - misc.JavaEarHandler , - - # is this redundant with Jar manifest? - misc.JavaJarHandler, - - misc.JavaWarHandler, - misc.JavaWarWebXmlHandler, - - misc.JBossSarHandler , - misc.JBossServiceXmlHandler , - - misc.MeteorPackageHandler, - misc.MozillaExtensionHandler, - misc.NsisInstallerHandler, - misc.SharArchiveHandler, - misc.SquashfsImageHandler, - npm.NpmPackageJsonHandler, - npm.NpmPackageLockJsonHandler, - npm.NpmShrinkwrapJsonHandler, - npm.YarnLockV1Handler, - npm.YarnLockV2Handler, - - nuget.NugetNupkgHandler, - nuget.NugetNuspecHandler, - - opam.OpamFileHandler, - - phpcomposer.PhpComposerJsonHandler, - phpcomposer.PhpComposerLockHandler, - - pubspec.DartPubspecYamlHandler, - pubspec.DartPubspecLockHandler, - - pypi.PipfileHandler, - pypi.PipfileLockHandler, - pypi.PipRequirementsFileHandler, - pypi.PypiEggHandler, - # pypi.PypiSdistArchiveHandler, - pypi.PypiWheelHandler, - pypi.PyprojectTomlHandler, - pypi.PythonEditableInstallationPkgInfoFile, - pypi.PythonEggPkgInfoFile, - pypi.PythonInstalledWheelMetadataFile, - pypi.PythonSdistPkgInfoFile, - pypi.PythonSetupPyHandler, - pypi.SetupCfgHandler, - - readme.ReadmeHandler, - - rpm.RpmArchiveHandler, - rpm.RpmSpecfileHandler, - - rubygems.GemMetadataArchiveExtractedHandler, - rubygems.GemArchiveHandler, - - # the order of these handlers matter - rubygems.GemfileInExtractedGemHandler, - rubygems.GemfileHandler, - - # the order of these handlers matter - rubygems.GemfileLockInExtractedGemHandler, - rubygems.GemfileLockHandler, - - # the order of these handlers matter - rubygems.GemspecInInstalledVendorBundleSpecificationsHandler, - rubygems.GemspecInExtractedGemHandler, - rubygems.GemspecHandler, - - windows.MicrosoftUpdateManifestHandler, - - win_pe.WindowsExecutableHandler, -] - -if on_linux: - APPLICATION_PACKAGE_DATAFILE_HANDLERS += [ - msi.MsiInstallerHandler, - ] - -SYSTEM_PACKAGE_DATAFILE_HANDLERS = [ - alpine.AlpineInstalledDatabaseHandler, - - debian_copyright.DebianCopyrightFileInPackageHandler, - debian_copyright.DebianCopyrightFileInSourceHandler, - - debian.DebianDistrolessInstalledDatabaseHandler, - - debian.DebianInstalledFilelistHandler, - debian.DebianInstalledMd5sumFilelistHandler, - debian.DebianInstalledStatusDatabaseHandler -] - -if on_linux: - SYSTEM_PACKAGE_DATAFILE_HANDLERS += [ - rpm.RpmInstalledBdbDatabaseHandler, - rpm.RpmInstalledSqliteDatabaseHandler, - rpm.RpmInstalledNdbDatabaseHandler, - - win_reg.InstalledProgramFromDockerSoftwareDeltaHandler, - win_reg.InstalledProgramFromDockerFilesSoftwareHandler, - win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler, - ] - -ALL_DATAFILE_HANDLERS = ( - APPLICATION_PACKAGE_DATAFILE_HANDLERS + [ - p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS - if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS - ] -) - -HANDLER_BY_DATASOURCE_ID = {handler.datasource_id: handler for handler in ALL_DATAFILE_HANDLERS} - - -class UnknownPackageDatasource(Exception): - pass - - -def get_package_handler(package_data): - """ - Return the DatafileHandler class that corresponds to a ``package_data`` - PackageData object. Raise a UnknownPackageDatasource error if the - DatafileHandler is not found. - """ - ppc = HANDLER_BY_DATASOURCE_ID.get(package_data.datasource_id) - if not ppc: - raise UnknownPackageDatasource(package_data) - return ppc - - -PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = { - maven.MavenPackageData.datasource_id: maven.MavenPackageData -} diff --git a/src/packagedcode/handlers.py b/src/packagedcode/handlers.py new file mode 100644 index 00000000000..a6d84c1e56e --- /dev/null +++ b/src/packagedcode/handlers.py @@ -0,0 +1,261 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import attr + +from commoncode.system import on_linux +from packagedcode import about +from packagedcode import alpine +from packagedcode import bower +from packagedcode import build +from packagedcode import build_gradle +from packagedcode import cargo +from packagedcode import chef +from packagedcode import debian +from packagedcode import debian_copyright +from packagedcode import distro +from packagedcode import conda +from packagedcode import conan +from packagedcode import cocoapods +from packagedcode import cran +from packagedcode import freebsd +from packagedcode import godeps +from packagedcode import golang +from packagedcode import haxe +from packagedcode import maven +from packagedcode import misc +from packagedcode import npm +from packagedcode import nuget +from packagedcode import opam +from packagedcode import phpcomposer +from packagedcode import pubspec +from packagedcode import pypi +from packagedcode import readme +from packagedcode import rpm +from packagedcode import rubygems +from packagedcode import win_pe +from packagedcode import windows + +if on_linux: + from packagedcode import msi + from packagedcode import win_reg + +# Note: the order matters: from the most to the least specific parser. +# a handler classes MUST be added to this list to be active +APPLICATION_PACKAGE_DATAFILE_HANDLERS = [ + about.AboutFileHandler, + + alpine.AlpineApkArchiveHandler, + alpine.AlpineApkbuildHandler, + + bower.BowerJsonHandler, + + build_gradle.BuildGradleHandler, + + build.AutotoolsConfigureHandler, + build.BazelBuildHandler, + build.BuckMetadataBzlHandler, + build.BuckPackageHandler, + + cargo.CargoLockHandler, + cargo.CargoTomlHandler, + + chef.ChefMetadataJsonHandler, + chef.ChefMetadataRbHandler, + + cocoapods.PodspecHandler, + cocoapods.PodspecJsonHandler, + cocoapods.PodfileLockHandler, + cocoapods.PodfileHandler, + + conda.CondaYamlHandler, + conda.CondaMetaYamlHandler, + + conan.ConanFileHandler, + conan.ConanDataHandler, + + cran.CranDescriptionFileHandler, + + debian_copyright.DebianCopyrightFileInPackageHandler, + debian_copyright.StandaloneDebianCopyrightFileHandler, + debian.DebianDscFileHandler, + + debian.DebianControlFileInExtractedDebHandler, + debian.DebianControlFileInSourceHandler, + + debian.DebianDebPackageHandler, + debian.DebianMd5sumFilelistInPackageHandler, + + debian.DebianSourcePackageMetadataTarballHandler, + debian.DebianSourcePackageTarballHandler, + + distro.EtcOsReleaseHandler, + + freebsd.CompactManifestHandler, + + godeps.GodepsHandler, + golang.GoModHandler, + golang.GoSumHandler, + + haxe.HaxelibJsonHandler, + + maven.MavenPomXmlHandler, + maven.MavenPomPropertiesHandler, + maven.JavaJarManifestHandler, + maven.JavaOSGiManifestHandler, + + misc.AndroidAppArchiveHandler, + misc.AndroidLibraryHandler, + misc.AppleDmgHandler, + misc.Axis2MarArchiveHandler , + misc.Axis2MarModuleXmlHandler , + misc.CabArchiveHandler, + misc.ChromeExtensionHandler, + misc.CpanDistIniHandler , + misc.CpanMakefilePlHandler, + misc.CpanManifestHandler, + misc.CpanMetaJsonHandler, + misc.CpanMetaYmlHandler, + misc.InstallShieldPackageHandler, + misc.IosAppIpaHandler, + misc.IsoImageHandler, + misc.IvyXmlHandler, + + misc.JavaEarAppXmlHandler , + misc.JavaEarHandler , + + # is this redundant with Jar manifest? + misc.JavaJarHandler, + + misc.JavaWarHandler, + misc.JavaWarWebXmlHandler, + + misc.JBossSarHandler , + misc.JBossServiceXmlHandler , + + misc.MeteorPackageHandler, + misc.MozillaExtensionHandler, + misc.NsisInstallerHandler, + misc.SharArchiveHandler, + misc.SquashfsImageHandler, + npm.NpmPackageJsonHandler, + npm.NpmPackageLockJsonHandler, + npm.NpmShrinkwrapJsonHandler, + npm.YarnLockV1Handler, + npm.YarnLockV2Handler, + + nuget.NugetNupkgHandler, + nuget.NugetNuspecHandler, + + opam.OpamFileHandler, + + phpcomposer.PhpComposerJsonHandler, + phpcomposer.PhpComposerLockHandler, + + pubspec.DartPubspecYamlHandler, + pubspec.DartPubspecLockHandler, + + pypi.PipfileHandler, + pypi.PipfileLockHandler, + pypi.PipRequirementsFileHandler, + pypi.PypiEggHandler, + # pypi.PypiSdistArchiveHandler, + pypi.PypiWheelHandler, + pypi.PyprojectTomlHandler, + pypi.PythonEditableInstallationPkgInfoFile, + pypi.PythonEggPkgInfoFile, + pypi.PythonInstalledWheelMetadataFile, + pypi.PythonSdistPkgInfoFile, + pypi.PythonSetupPyHandler, + pypi.SetupCfgHandler, + + readme.ReadmeHandler, + + rpm.RpmArchiveHandler, + rpm.RpmSpecfileHandler, + + rubygems.GemMetadataArchiveExtractedHandler, + rubygems.GemArchiveHandler, + + # the order of these handlers matter + rubygems.GemfileInExtractedGemHandler, + rubygems.GemfileHandler, + + # the order of these handlers matter + rubygems.GemfileLockInExtractedGemHandler, + rubygems.GemfileLockHandler, + + # the order of these handlers matter + rubygems.GemspecInInstalledVendorBundleSpecificationsHandler, + rubygems.GemspecInExtractedGemHandler, + rubygems.GemspecHandler, + + windows.MicrosoftUpdateManifestHandler, + + win_pe.WindowsExecutableHandler, +] + +if on_linux: + APPLICATION_PACKAGE_DATAFILE_HANDLERS += [ + msi.MsiInstallerHandler, + ] + +SYSTEM_PACKAGE_DATAFILE_HANDLERS = [ + alpine.AlpineInstalledDatabaseHandler, + + debian_copyright.DebianCopyrightFileInPackageHandler, + debian_copyright.DebianCopyrightFileInSourceHandler, + + debian.DebianDistrolessInstalledDatabaseHandler, + + debian.DebianInstalledFilelistHandler, + debian.DebianInstalledMd5sumFilelistHandler, + debian.DebianInstalledStatusDatabaseHandler +] + +if on_linux: + SYSTEM_PACKAGE_DATAFILE_HANDLERS += [ + rpm.RpmInstalledBdbDatabaseHandler, + rpm.RpmInstalledSqliteDatabaseHandler, + rpm.RpmInstalledNdbDatabaseHandler, + + win_reg.InstalledProgramFromDockerSoftwareDeltaHandler, + win_reg.InstalledProgramFromDockerFilesSoftwareHandler, + win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler, + ] + +ALL_DATAFILE_HANDLERS = ( + APPLICATION_PACKAGE_DATAFILE_HANDLERS + [ + p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS + if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS + ] +) + +HANDLER_BY_DATASOURCE_ID = {handler.datasource_id: handler for handler in ALL_DATAFILE_HANDLERS} + + +class UnknownPackageDatasource(Exception): + pass + + +def get_package_handler(package_data): + """ + Return the DatafileHandler class that corresponds to a ``package_data`` + PackageData object. Raise a UnknownPackageDatasource error if the + DatafileHandler is not found. + """ + ppc = HANDLER_BY_DATASOURCE_ID.get(package_data.datasource_id) + if not ppc: + raise UnknownPackageDatasource(package_data) + return ppc + + +PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = { + maven.MavenPackageData.datasource_id: maven.MavenPackageData +} diff --git a/src/packagedcode/licensing.py b/src/packagedcode/licensing.py index d6399626463..45cca105ace 100644 --- a/src/packagedcode/licensing.py +++ b/src/packagedcode/licensing.py @@ -837,7 +837,7 @@ def get_license_detections_and_expression( Return None if the `query_string` is empty. Return "unknown" as a license expression if there is a `query_string` but nothing was detected. """ - from packagedcode import PACKAGE_DATA_CLASS_BY_DATASOURCE_ID + from packagedcode.handlers import PACKAGE_DATA_CLASS_BY_DATASOURCE_ID detection_data = [] license_expression = None diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 50a181214ba..7dc88aea916 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -103,9 +103,9 @@ dependencies. When implementing a new package type and manifest file format, subclass DatafileHandler and implement the parse() and assemble() methods for this package datafile format and package type. Then register this class in - ``packagedcode.APPLICATION_PACKAGE_DATAFILE_HANDLERS`` if this is an - application package or ``packagedcode.SYSTEM_PACKAGE_DATAFILE_HANDLERS`` if - this is a system package. + ``packagedcode.handlers.APPLICATION_PACKAGE_DATAFILE_HANDLERS`` if this is an + application package or ``packagedcode.handlers.SYSTEM_PACKAGE_DATAFILE_HANDLERS`` + if this is a system package. Beyond these we have a few secondary models: @@ -743,7 +743,7 @@ def from_data(cls, package_data, package_only=False): @property def can_assemble(self): - from packagedcode import HANDLER_BY_DATASOURCE_ID + from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID handler = HANDLER_BY_DATASOURCE_ID.get(self.datasource_id) if issubclass(handler, NonAssemblableDatafileHandler): return False @@ -950,7 +950,7 @@ def get_license_detections_and_expression(self): def get_default_relation_license(datasource_id): - from packagedcode import HANDLER_BY_DATASOURCE_ID + from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID handler = HANDLER_BY_DATASOURCE_ID[datasource_id] return handler.default_relation_license diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index b1487022c40..0def4cb44b1 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -26,7 +26,7 @@ from licensedcode.cache import get_cache from licensedcode.detection import DetectionRule from licensedcode.detection import populate_matches_with_path -from packagedcode import get_package_handler +from packagedcode.handlers import get_package_handler from packagedcode.licensing import add_referenced_license_matches_for_package from packagedcode.licensing import add_referenced_license_detection_from_package from packagedcode.licensing import add_license_from_sibling_file @@ -78,7 +78,7 @@ def print_packages(ctx, param, value): def get_available_package_parsers(docs=False): - from packagedcode import ALL_DATAFILE_HANDLERS + from packagedcode.handlers import ALL_DATAFILE_HANDLERS all_data_packages = [] diff --git a/src/packagedcode/recognize.py b/src/packagedcode/recognize.py index 9dd7c643482..3a67afa6ab3 100644 --- a/src/packagedcode/recognize.py +++ b/src/packagedcode/recognize.py @@ -11,9 +11,9 @@ import sys from commoncode import filetype -from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS -from packagedcode import SYSTEM_PACKAGE_DATAFILE_HANDLERS -from packagedcode import ALL_DATAFILE_HANDLERS +from packagedcode.handlers import APPLICATION_PACKAGE_DATAFILE_HANDLERS +from packagedcode.handlers import SYSTEM_PACKAGE_DATAFILE_HANDLERS +from packagedcode.handlers import ALL_DATAFILE_HANDLERS from packagedcode import models TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False) diff --git a/src/summarycode/classify_plugin.py b/src/summarycode/classify_plugin.py index 59e46da70dc..a5989497099 100644 --- a/src/summarycode/classify_plugin.py +++ b/src/summarycode/classify_plugin.py @@ -13,7 +13,7 @@ from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import POST_SCAN_GROUP -from packagedcode import get_package_handler +from packagedcode.handlers import get_package_handler from packagedcode.models import PackageData from summarycode.classify import set_classification_flags diff --git a/tests/packagedcode/packages_test_utils.py b/tests/packagedcode/packages_test_utils.py index ba25c20b3fb..f865f66e3ea 100644 --- a/tests/packagedcode/packages_test_utils.py +++ b/tests/packagedcode/packages_test_utils.py @@ -81,7 +81,7 @@ def check_packages_data( def populate_license_fields(package_data): if package_data.extracted_license_statement and not package_data.declared_license_expression: - from packagedcode import HANDLER_BY_DATASOURCE_ID + from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID handler = HANDLER_BY_DATASOURCE_ID[package_data.datasource_id] handler.populate_license_fields(package_data) diff --git a/tests/packagedcode/test_package_models.py b/tests/packagedcode/test_package_models.py index d518bd60f84..76f735def48 100644 --- a/tests/packagedcode/test_package_models.py +++ b/tests/packagedcode/test_package_models.py @@ -11,7 +11,7 @@ from packagedcode import misc from packagedcode import models -from packagedcode import ALL_DATAFILE_HANDLERS +from packagedcode.handlers import ALL_DATAFILE_HANDLERS from packagedcode.models import PackageData from packagedcode.models import Party from packagedcode.plugin_package import get_package_and_deps From 70c511666678e3ce3eebe2e6c5d2962c80edd5ff Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 09:19:16 +0200 Subject: [PATCH 18/28] Split BSD-licensed code in own module This is cleaner and makes tracking licenses much easier. Signed-off-by: Philippe Ombredanne --- src/packagedcode/pypi.py | 63 ++------------- src/packagedcode/pypi_utils.py | 81 +++++++++++++++++++ .../{pypi.py.ABOUT => pypi_utils.py.ABOUT} | 9 +-- .../{pypi.py.NOTICE => pypi_utils.py.LICENSE} | 9 --- 4 files changed, 90 insertions(+), 72 deletions(-) create mode 100644 src/packagedcode/pypi_utils.py rename src/packagedcode/{pypi.py.ABOUT => pypi_utils.py.ABOUT} (61%) rename src/packagedcode/{pypi.py.NOTICE => pypi_utils.py.LICENSE} (81%) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 33f2ec394ea..c923568c6e0 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -17,6 +17,7 @@ import sys import tempfile import zipfile + from configparser import ConfigParser from pathlib import Path from typing import NamedTuple @@ -27,6 +28,7 @@ import packvers as packaging import pip_requirements_parser import pkginfo2 + from commoncode import fileutils from commoncode.fileutils import as_posixpath from commoncode.resource import Resource @@ -40,6 +42,9 @@ from packagedcode.utils import build_description from packagedcode.utils import yield_dependencies_from_package_data from packagedcode.utils import yield_dependencies_from_package_resource +from packagedcode.pypi_utils import find_pattern +from packagedcode.pypi_utils import find_setup_py_dunder_version +from packagedcode.pypi_utils import find_dunder_version try: from zipfile import Path as ZipPath @@ -1757,41 +1762,6 @@ def add_url(_url, _utype=None, _attribute=None): return urls, extra_data -def find_pattern(location, pattern): - """ - Search the file at `location` for a patern regex on a single line and return - this or None if not found. Reads the supplied location as text without - importing it. - - Code inspired and heavily modified from: - https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py#L34 - SPDX-License-Identifier: BSD-3-Clause - (C) 2001-2020 Chris Liechti - """ - with open(location) as fp: - content = fp.read() - - match = re.search(pattern, content) - if match: - return match.group(1).strip() - - -def find_dunder_version(location): - """ - Return a "dunder" __version__ string or None from searching the module file - at `location`. - - Code inspired and heavily modified from: - https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py#L34 - SPDX-License-Identifier: BSD-3-Clause - (C) 2001-2020 Chris Liechti - """ - pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) - match = find_pattern(location, pattern) - if TRACE: logger_debug('find_dunder_version:', 'location:', location, 'match:', match) - return match - - def find_plain_version(location): """ Return a plain version attribute string or None from searching the module @@ -1803,29 +1773,6 @@ def find_plain_version(location): return match -def find_setup_py_dunder_version(location): - """ - Return a "dunder" __version__ expression string used as a setup(version) - argument or None from searching the setup.py file at `location`. - - For instance: - setup( - version=six.__version__, - ... - would return six.__version__ - - Code inspired and heavily modified from: - https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py#L34 - SPDX-License-Identifier: BSD-3-Clause - (C) 2001-2020 Chris Liechti - """ - pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.MULTILINE) - match = find_pattern(location, pattern) - if TRACE: - logger_debug('find_setup_py_dunder_version:', 'location:', location, 'match:', match) - return match - - def detect_version_attribute(setup_location): """ Return a detected version from a setup.py file at `location` if used as in diff --git a/src/packagedcode/pypi_utils.py b/src/packagedcode/pypi_utils.py new file mode 100644 index 00000000000..dfa7eda3db0 --- /dev/null +++ b/src/packagedcode/pypi_utils.py @@ -0,0 +1,81 @@ +# +# Copyright (c) nexB Inc., Chris Liechti and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: BSD-3-Clause +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import os +import re +import sys + +""" +Utilities to find dunder variables in Python code. + +The functions find_pattern(), find_dunder_version(), find_setup_py_dunder_version() are inspired +and heavily modified from Pyserial setup.py + +https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py +""" + +TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False) + + +def logger_debug(*args): + pass + + +logger = logging.getLogger(__name__) + +if TRACE: + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + + +def find_pattern(location, pattern): + """ + Search the file at `location` for a patern regex on a single line and return + this or None if not found. Reads the supplied location as text without + importing it. + + """ + with open(location) as fp: + content = fp.read() + + match = re.search(pattern, content) + if match: + return match.group(1).strip() + + +def find_dunder_version(location): + """ + Return a "dunder" __version__ string or None from searching the module file + at `location`. + """ + pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) + match = find_pattern(location, pattern) + if TRACE: logger_debug('find_dunder_version:', 'location:', location, 'match:', match) + return match + + +def find_setup_py_dunder_version(location): + """ + Return a "dunder" __version__ expression string used as a setup(version) + argument or None from searching the setup.py file at `location`. + + For instance: + setup( + version=six.__version__, + ... + would return six.__version__ + """ + pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.MULTILINE) + match = find_pattern(location, pattern) + if TRACE: + logger_debug('find_setup_py_dunder_version:', 'location:', location, 'match:', match) + return match diff --git a/src/packagedcode/pypi.py.ABOUT b/src/packagedcode/pypi_utils.py.ABOUT similarity index 61% rename from src/packagedcode/pypi.py.ABOUT rename to src/packagedcode/pypi_utils.py.ABOUT index aa64a4d0617..104f4ac2807 100644 --- a/src/packagedcode/pypi.py.ABOUT +++ b/src/packagedcode/pypi_utils.py.ABOUT @@ -1,4 +1,4 @@ -about_resource: pypi.py +about_resource: pypi_utils.py name: pyserial namespace: pyserial package_url: pkg:github/pyserial/pyserial@d867871e6aa33301#setup.py @@ -6,8 +6,7 @@ attribute: yes copyright: Copyright (c) 2001-2020 Chris Liechti download_url: https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py#L34 license_expression: bsd-new -notice_file: pypi.py.NOTICE +license_file: pypi_utils.py.LICENSE primary_language: Python -notes: The functions find_pattern(), find_dunder_version(), - find_setup_py_dunder_version() are inspired and heavily modified from Pyserial - setup.py +notes: The functions find_pattern(), find_dunder_version(), find_setup_py_dunder_version() are + inspired and heavily modified from Pyserial setup.py diff --git a/src/packagedcode/pypi.py.NOTICE b/src/packagedcode/pypi_utils.py.LICENSE similarity index 81% rename from src/packagedcode/pypi.py.NOTICE rename to src/packagedcode/pypi_utils.py.LICENSE index 6eb40f32ee3..20905611ce3 100644 --- a/src/packagedcode/pypi.py.NOTICE +++ b/src/packagedcode/pypi_utils.py.LICENSE @@ -28,12 +28,3 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---------------------------------------------------------------------------- -Note: -Individual files contain the following tag instead of the full license text. - - SPDX-License-Identifier: BSD-3-Clause - -This enables machine processing of license information based on the SPDX -License Identifiers that are here available: http://spdx.org/licenses/ \ No newline at end of file From 7e0e5b5b8d27ce1a2e0e482dd958b639e1a3432f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 11:01:32 +0200 Subject: [PATCH 19/28] Improve formattings and docstrings Signed-off-by: Philippe Ombredanne --- src/packagedcode/licensing.py | 66 ++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/packagedcode/licensing.py b/src/packagedcode/licensing.py index 45cca105ace..32956029bc8 100644 --- a/src/packagedcode/licensing.py +++ b/src/packagedcode/licensing.py @@ -37,7 +37,6 @@ import saneyaml - """ Detect and normalize licenses as found in package manifests data. """ @@ -248,7 +247,7 @@ def add_referenced_license_detection_from_package(resource, codebase): continue for sibling_detection in sibling_license_detections: - + modified = True detection_modified = True license_match_mappings.extend(sibling_detection["matches"]) @@ -591,7 +590,7 @@ def matches_have_unknown(matches, licensing=Licensing()): def get_license_detections_from_matches(matches): """ - Return a list of LicenseDetection objects given a list of LicenseMatch objects. + Return a list of LicenseDetection given a ``matches`` list of LicenseMatch. """ license_detections = [] @@ -624,7 +623,7 @@ def get_license_expression_from_detections(license_detections, relation='AND', u license_expression = str(license_expressions[0]) else: license_expression = str( - combine_expressions(license_expressions, relation=relation, unique=unique) + combine_expressions(expressions=license_expressions, relation=relation, unique=unique) ) return license_expression @@ -639,8 +638,9 @@ def get_mapping_and_expression_from_detections( whole_lines=True, ): """ - Return a list of LicenseDetection data mapping and a license_expression - from a list of LicenseDetection objects. + Return a tuple of: + (list of LicenseDetection data mapping , license_expression string) + from a ``license_detections`` list of LicenseDetection. """ detection_data = [] @@ -672,13 +672,16 @@ def get_mapping_and_expression_from_detections( return detection_data, license_expression +# TODO: fixme do not use a double negative def is_declared_license_not_fully_matched(matches): """ - verify that we consumed 100% of the query string e.g. that we - have no unknown leftover. + Return True if we DID NOT we consumed all the query string e.g. that we have no unknown leftover. - # 1. have all matches 100% coverage? - # 2. is declared license fully matched? + A declared license statement is supposed to be entreily about licensing and therefore we should + detect everything in its string. + We check if: + - do all matches have a 100% coverage? + - is the declared license fully matched? """ all_matches_have_full_coverage = all(m.coverage() == 100 for m in matches) @@ -690,7 +693,7 @@ def is_declared_license_not_fully_matched(matches): raise Exception( 'Inconsistent package.extracted_license_statement: text with multiple "queries".' 'Please report this issue to the scancode-toolkit team.\n' - f'{matches}' + f'{matches!r}' ) query_len = len(query.tokens) @@ -712,8 +715,8 @@ def get_normalized_license_detections( expression_symbols=None, ): """ - Return a normalized license expression string detected from a list of - declared license items. + Return a normalized license expression string detected from a ``extracted_license`` list of + declared license items or string or dict """ license_detections = [] @@ -736,6 +739,7 @@ def get_normalized_license_detections( logger_debug(f'get_normalized_license_detections: str:') elif isinstance(extracted_license, dict): + # detect first on values, not keys for extracted_license_statement in extracted_license.values(): detections = get_license_detections_for_extracted_license_statement( extracted_license_statement=extracted_license_statement, @@ -744,17 +748,27 @@ def get_normalized_license_detections( expression_symbols=expression_symbols, ) if TRACE: - logger_debug(f'get_normalized_license_detections: dict: extracted_license_statement: {extracted_license_statement}: detections: {detections}') + logger_debug( + 'get_normalized_license_detections: dict: ' + f'extracted_license_statement: {extracted_license_statement}: ' + f'detections: {detections}' + ) if detections: license_detections.extend(detections) if not license_detections: + # try again first on items if nothing was found unknown_dict_object = repr(dict(extracted_license.items())) unknown_detection = get_unknown_license_detection(query_string=unknown_dict_object) license_detections.append(unknown_detection) if TRACE: - logger_debug(f'get_normalized_license_detections: dict: unknown_dict_object: {unknown_dict_object}, unknown_detection: {saneyaml.dump(unknown_detection.to_dict())}') + logger_debug( + 'get_normalized_license_detections: dict: ' + f'unknown_dict_object: {unknown_dict_object}, ' + f'unknown_detection: {saneyaml.dump(unknown_detection.to_dict())}' + ) + else: extracted_license_statement = saneyaml.dump(extracted_license) license_detections = get_license_detections_for_extracted_license_statement( @@ -781,7 +795,10 @@ def get_normalized_license_detections( license_detections.extend(detections) if TRACE: - logger_debug(f'get_normalized_license_detections: list(str): extracted_license_item: {extracted_license_item}: detections: {license_detections}') + logger_debug( + 'get_normalized_license_detections: list(str): ' + f'extracted_license_item: {extracted_license_item}: ' + f'detections: {license_detections}') elif isinstance(extracted_license_item, dict): for extracted_license_statement in extracted_license_item.values(): @@ -792,7 +809,11 @@ def get_normalized_license_detections( expression_symbols=expression_symbols, ) if TRACE: - logger_debug(f'get_normalized_license_detections: list(dict): extracted_license_statement: {extracted_license_statement}: detections: {detections}') + logger_debug( + 'get_normalized_license_detections: list(dict): ' + f'extracted_license_statement: {extracted_license_statement}: ' + f'detections: {detections}' + ) if detections: license_detections.extend(detections) @@ -810,7 +831,11 @@ def get_normalized_license_detections( license_detections.extend(detections) if TRACE: - logger_debug(f'get_normalized_license_detections: list(other): extracted_license_statement: {extracted_license_statement}: detections: {license_detections}') + logger_debug( + 'get_normalized_license_detections: list(other): ' + f'extracted_license_statement: {extracted_license_statement}: ' + f'detections: {license_detections}' + ) return license_detections @@ -821,7 +846,7 @@ def get_license_detections_and_expression( try_as_expression=True, approximate=True, expression_symbols=None, - datasource_id = None, + datasource_id=None, ): """ Given a text `extracted_license_statement` return a list of LicenseDetection objects. @@ -886,8 +911,7 @@ def get_license_detections_for_extracted_license_statement( expression_symbols=None, ): """ - Return a list of LicenseDetection objects after detecting licenses in - the given `extracted_license_statement`. + Detect licenses in an ``extracted_license_statement`` string and return a LicenseDetection list. """ if not extracted_license_statement: return [] From 451d56a7299fce42a3be915c98b8ffd4c7562353 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 11:05:00 +0200 Subject: [PATCH 20/28] Guard imports of external packages Guard import with "try/except ImportError" towards packaging models as a standalone wheel Signed-off-by: Philippe Ombredanne --- src/packagedcode/models.py | 181 +++++++++++++++++++++---------------- 1 file changed, 103 insertions(+), 78 deletions(-) diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 7dc88aea916..d03c26f7077 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -9,14 +9,16 @@ import os import uuid -from fnmatch import fnmatchcase import logging import sys +from fnmatch import fnmatchcase + import attr +import saneyaml + from packageurl import normalize_qualifiers from packageurl import PackageURL -import saneyaml from commoncode import filetype from commoncode.datautils import choices @@ -28,18 +30,12 @@ from commoncode.datautils import String from commoncode.fileutils import as_posixpath from commoncode.resource import Resource + try: from typecode import contenttype except ImportError: contenttype = None -try: - from packagedcode import licensing -except ImportError: - licensing = None - -from packagedcode.licensing import get_declared_license_expression_spdx - """ This module contain data models for package and dependencies, abstracting and normalizing the small differences that exist across different package types @@ -61,7 +57,7 @@ - "metadata" such as a name, version or description, - "dependencies" on other packages either potential with version requirements or - resolved and locked with concrete versions), and + resolved and locked with concrete versions, and - "build" and packaging scripts and instructions. @@ -737,42 +733,32 @@ def from_data(cls, package_data, package_only=False): package_data.populate_license_fields() package_data.populate_holder_field() else: - package_data.normalize_extracted_license_statement() + package_data.extracted_license_statement = normalize_extracted_license_statement( + package_data.extracted_license_statement + ) return package_data @property def can_assemble(self): - from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID - handler = HANDLER_BY_DATASOURCE_ID.get(self.datasource_id) - if issubclass(handler, NonAssemblableDatafileHandler): - return False - - return True + try: + from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID + handler = HANDLER_BY_DATASOURCE_ID.get(self.datasource_id) + if issubclass(handler, NonAssemblableDatafileHandler): + return False - def normalize_extracted_license_statement(self): - """ - Normalizes the extracted license statement to a readable - YAML string if it was a pythonic object. - """ - if ( - self.extracted_license_statement and - not isinstance(self.extracted_license_statement, str) - ): - if isinstance(self.extracted_license_statement, dict): - self.extracted_license_statement = saneyaml.dump( - dict(self.extracted_license_statement.items()) - ) - else: - self.extracted_license_statement = saneyaml.dump( - self.extracted_license_statement - ) + return True + except ImportError: + return False def populate_holder_field(self): if not self.copyright: return - from cluecode.copyrights import CopyrightDetector + try: + from cluecode.copyrights import CopyrightDetector + except ImportError: + return numbered_lines = list(enumerate(self.copyright.split("\n"), start=1)) detector = CopyrightDetector() @@ -811,22 +797,29 @@ def populate_license_fields(self): """ if not self.declared_license_expression and self.extracted_license_statement: - self.license_detections, self.declared_license_expression = \ - self.get_license_detections_and_expression() + try: + from packagedcode.licensing import get_declared_license_expression_spdx - self.declared_license_expression_spdx = get_declared_license_expression_spdx( - declared_license_expression=self.declared_license_expression - ) + lds, dle = self.get_license_detections_and_expression() + self.license_detections = lds + self.declared_license_expression = dle - if TRACE: - logger_debug( - f"PackageData: populate_license_fields:" - f"extracted_license_statement: {self.extracted_license_statement}" - f"declared_license_expression: {self.declared_license_expression}" - f"license_detections: {self.license_detections}" - ) + dles = get_declared_license_expression_spdx(declared_license_expression=dle) + self.declared_license_expression_spdx = dles - self.normalize_extracted_license_statement() + if TRACE: + logger_debug( + f"PackageData: populate_license_fields:" + f"extracted_license_statement: {self.extracted_license_statement}" + f"declared_license_expression: {self.declared_license_expression}" + f"license_detections: {self.license_detections}" + ) + except ImportError: + pass + + self.extracted_license_statement = normalize_extracted_license_statement( + self.extracted_license_statement + ) def update_purl_fields(self, package_data, replace=False): @@ -930,29 +923,53 @@ def get_license_detections_and_expression(self): Called only when using the default assemble() implementation. Subclass can override as needed. """ - from packagedcode.licensing import get_license_detections_and_expression if not self.extracted_license_statement: return [], None - if self.datasource_id: - default_relation_license=get_default_relation_license( + try: + from packagedcode.licensing import get_license_detections_and_expression + + if self.datasource_id: + default_relation_license = get_default_relation_license( + datasource_id=self.datasource_id, + ) + else: + default_relation_license = 'AND' + + return get_license_detections_and_expression( + extracted_license_statement=self.extracted_license_statement, + default_relation_license=default_relation_license, datasource_id=self.datasource_id, ) - else: - default_relation_license = 'AND' + except ImportError: + return [], None - return get_license_detections_and_expression( - extracted_license_statement=self.extracted_license_statement, - default_relation_license=default_relation_license, - datasource_id=self.datasource_id, - ) + +def normalize_extracted_license_statement(els): + """ + Normalizes the extracted license statement ``els`` to a readable + YAML string if it was a pythonic object. + """ + if (els and not isinstance(els, str)): + if isinstance(els, dict): + els = saneyaml.dump(dict(els.items())) + else: + els = saneyaml.dump(els) + return els def get_default_relation_license(datasource_id): - from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID - handler = HANDLER_BY_DATASOURCE_ID[datasource_id] - return handler.default_relation_license + """ + Return the default relationship between multiple license statements (OR or AND) defaulting to + "AND". + """ + try: + from packagedcode.handlers import HANDLER_BY_DATASOURCE_ID + handler = HANDLER_BY_DATASOURCE_ID[datasource_id] + return handler.default_relation_license + except ImportError: + return def _rehydrate_list(cls, values): @@ -1434,25 +1451,33 @@ def populate_license_fields(cls, package_data): object, and add the declared_license_expression (and the spdx expression) and corresponding LicenseDetection data. """ - if not package_data.declared_license_expression and package_data.extracted_license_statement: + pd_els = package_data.extracted_license_statement + if not package_data.declared_license_expression and pd_els: - package_data.license_detections, package_data.declared_license_expression = \ - cls.get_license_detections_and_expression(package_data) + try: + from packagedcode.licensing import get_declared_license_expression_spdx + pd_ld, pd_dle = cls.get_license_detections_and_expression(package_data) + package_data.license_detections = pd_ld + package_data.declared_license_expression = pd_dle - package_data.declared_license_expression_spdx = get_declared_license_expression_spdx( - declared_license_expression=package_data.declared_license_expression - ) + pd_dles = get_declared_license_expression_spdx(declared_license_expression=pd_dle) + package_data.declared_license_expression_spdx = pd_dles - if TRACE: - logger_debug( - f"DatafileHandler: populate_license_fields:" - f"extracted_license_statement: {package_data.extracted_license_statement}" - f"declared_license_expression: {package_data.declared_license_expression}" - f"license_detections: {package_data.license_detections}" - ) + if TRACE: + logger_debug( + f"DatafileHandler: populate_license_fields:" + f"extracted_license_statement: {package_data.extracted_license_statement}" + f"declared_license_expression: {package_data.declared_license_expression}" + f"license_detections: {package_data.license_detections}" + ) - if package_data.extracted_license_statement and not isinstance(package_data.extracted_license_statement, str): - package_data.extracted_license_statement = repr(package_data.extracted_license_statement) + except ImportError: + pass + + if pd_els and not isinstance(pd_els, str): + package_data.extracted_license_statement = repr(pd_els) + # TODO: consider using a common normalization of the extracted license field + # package_data.extracted_license_statement = normalize_extracted_license_statement(pd_els) @classmethod def get_top_level_resources(cls, manifest_resource, codebase): @@ -1571,10 +1596,10 @@ def from_package_data(cls, package_data, datafile_path, package_only=False): license_match['from_file'] = datafile_path package = cls.from_dict(package_data_mapping) - + if not package.package_uid: package.package_uid = build_package_uid(package.purl) - + if not package_only: package.populate_license_fields() package.populate_holder_field() From 46cb7e8dda2cc1fccf333573b437877496c0fc5c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 15:41:44 +0200 Subject: [PATCH 21/28] Remove unused import Signed-off-by: Philippe Ombredanne --- src/packagedcode/handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/packagedcode/handlers.py b/src/packagedcode/handlers.py index a6d84c1e56e..195dede1bf8 100644 --- a/src/packagedcode/handlers.py +++ b/src/packagedcode/handlers.py @@ -7,8 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import attr - from commoncode.system import on_linux from packagedcode import about from packagedcode import alpine From 8d780b77fa35cebcbdc17c6a76040340ea7a6f37 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 15:46:13 +0200 Subject: [PATCH 22/28] Package packagedcode as two wheels * packagedcode-models only contains the models.py * packagedcode-pypi only contains the pypi code and depends on models. This way we have small minimal wheel needed to support FetchCode and python-inspector. Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- pyproject-packagedcode-models.toml | 41 +----------- pyproject-packagedcode-pypi.toml | 103 +++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 pyproject-packagedcode-pypi.toml diff --git a/pyproject-packagedcode-models.toml b/pyproject-packagedcode-models.toml index cc901764150..e538776e7c8 100644 --- a/pyproject-packagedcode-models.toml +++ b/pyproject-packagedcode-models.toml @@ -6,7 +6,7 @@ build-backend = "flot.buildapi" name = "packagedcode-models" version = "32.1.0" description = "PackagedCode Models is a library with core models of package manifests, lockfiles and dependencies for all type of app and system packages keyed by PURL." -license = { text = "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" } +license = { text = "Apache-2.0" } # TODO consider a readme? # readme = "README.rst" requires-python = ">=3.8" @@ -41,35 +41,10 @@ classifiers = [ dependencies = [ "attrs >= 18.1,!=20.1.0;python_version<'3.11'", "attrs >= 22.1.0;python_version>='3.11'", - "boolean.py >= 4.0", - "click >= 6.7, !=7.0", "commoncode >= 31.0.3", - "container-inspector >= 31.0.0", - "debian-inspector >= 31.1.0", - "dparse2 >= 0.7.0", - "ftfy >= 6.0.0", - "gemfileparser2 >= 0.9.0", - "importlib_metadata", - "javaproperties >= 0.5", - "jinja2 >= 2.7.0", - "license_expression >= 30.1.1", - "lxml >= 4.9.2", "packageurl_python >= 0.9.0", - "packvers >= 21.0.0", - # use temp advanced patched release - "parameter-expansion-patched >= 0.3.1", - "pefile >= 2020.1.1", - "pip-requirements-parser >= 32.0.1", - "pkginfo2 >= 30.0.0", - "plugincode >= 32.0.0", - "pygmars >= 0.7.0", - "pygments", - "pymaven_patch >= 0.2.8", "saneyaml >= 0.6.0", - "toml >= 0.10.0", - "xmltodict >= 0.11.0", "typecode >= 30.0.1", - "typecode[full] >= 30.0.1", ] @@ -78,19 +53,9 @@ Homepage = "https://github.com/nexB/scancode-toolkit" [project.optional-dependencies] -# linux-only package handling -packages = [ - "rpm_inspector_rpm >= 4.16.1.3; platform_system == 'Linux'", - "regipy >= 3.1.0; platform_system == 'Linux'", - "packagedcode_msitools >= 0.101.210706; platform_system == 'Linux'", -] full = [ -# TODO: we still need all these deps and clenup is needed -#from licensedcode -#from summarycode -#from textcode -#from typecode + "typecode[full] >= 30.0.1", ] @@ -98,6 +63,7 @@ full = [ wheel_path_prefixes_to_strip = [ "src" ] includes = [ + "src/packagedcode/__init__.py", "src/packagedcode/models.py", ] @@ -131,6 +97,5 @@ sdist_extra_includes = [ "*.LICENSE", "NOTICE", "*.rst", - "requirements*", ] diff --git a/pyproject-packagedcode-pypi.toml b/pyproject-packagedcode-pypi.toml new file mode 100644 index 00000000000..5668f4266ed --- /dev/null +++ b/pyproject-packagedcode-pypi.toml @@ -0,0 +1,103 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "packagedcode-pypi" +version = "32.1.0" +description = "PackagedCode PyPI is a library to parse Python package manifests and lockfiles keyed by PURL." +license = { text = "Apache-2.0 AND BSD-3-Clause AND MIT" } +# TODO consider a readme? +# readme = "README.rst" +requires-python = ">=3.8" +authors = [ + { name = "nexB. Inc. and others", email = "info@aboutcode.org" }, +] + +keywords = [ + "open source", + "scan", + "package", + "dependency", + "sca", + "SBOM", + "package-url", + "purl", + "pypi", + "poetry", + "pip", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "packagedcode-models", + "attrs >= 18.1,!=20.1.0;python_version<'3.11'", + "attrs >= 22.1.0;python_version>='3.11'", + "commoncode >= 31.0.3", + "dparse2 >= 0.7.0", + "importlib_metadata", + "packageurl_python >= 0.9.0", + "packvers >= 21.0.0", + "pip-requirements-parser >= 32.0.1", + "pkginfo2 >= 30.0.0", + "zipp >= 3.0.0; python_version < \"3.9\"", +] + + +[project.urls] +Homepage = "https://github.com/nexB/scancode-toolkit" + + +[tool.flot] +wheel_path_prefixes_to_strip = [ "src" ] + +includes = [ + "src/packagedcode/__init__.py", + "src/packagedcode/pypi*.py", +] + +excludes = [ + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", +] + +metadata_files = [ + "*.LICENSE", + "NOTICE", + "AUTHORS.rst", + "CHANGELOG.rst", + "CODE_OF_CONDUCT.rst", +] + +editable_paths = [ "src", "tests" ] + +sdist_extra_includes = [ + "tests/packagedcode/*", + ".gitattributes", + ".gitignore", + "*.LICENSE", + "NOTICE", + "*.rst", + "requirements*", +] From 94f32cea77bd6d68c12a476fbbc20d2b115cdfe0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 16:03:06 +0200 Subject: [PATCH 23/28] Position comment before the commented section Signed-off-by: Philippe Ombredanne --- src/packagedcode/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index d03c26f7077..d3fd962a190 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -1475,9 +1475,9 @@ def populate_license_fields(cls, package_data): pass if pd_els and not isinstance(pd_els, str): - package_data.extracted_license_statement = repr(pd_els) # TODO: consider using a common normalization of the extracted license field # package_data.extracted_license_statement = normalize_extracted_license_statement(pd_els) + package_data.extracted_license_statement = repr(pd_els) @classmethod def get_top_level_resources(cls, manifest_resource, codebase): From b33a6fc9076add4b2d2a6a8aec064ee954c17e6a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 16:06:32 +0200 Subject: [PATCH 24/28] Build scancode and packagedcode for Pypi * Use new pyproject TOML fiels for scancode * Build packagedcode-models and packagedcode-pypi Reference: https://github.com/nexB/scancode-toolkit/issues/3459 Signed-off-by: Philippe Ombredanne --- etc/release/scancode-create-pypi-wheel-and-sdist.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/etc/release/scancode-create-pypi-wheel-and-sdist.sh b/etc/release/scancode-create-pypi-wheel-and-sdist.sh index 6bea3755860..f3cf797c416 100755 --- a/etc/release/scancode-create-pypi-wheel-and-sdist.sh +++ b/etc/release/scancode-create-pypi-wheel-and-sdist.sh @@ -25,9 +25,10 @@ venv/bin/flot --pyproject pyproject-licensedcode-data.toml --wheel --sdist venv/bin/flot --pyproject pyproject-licensedcode-index.toml --wheel --sdist # build code packages -venv/bin/flot --pyproject pyproject.toml --wheel --sdist -venv/bin/flot --pyproject pyproject-mini.toml --wheel --sdist -venv/bin/flot --pyproject pyproject-packagedcode.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-scancode-toolkit.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-scancode-toolkit-mini.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-packagedcode-models.toml --wheel --sdist +venv/bin/flot --pyproject pyproject-packagedcode-pypi.toml --wheel --sdist venv/bin/twine check dist/* From c64d47da49b1e97e6dd7d41470a067368e7202c5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 16:14:10 +0200 Subject: [PATCH 25/28] Import prospectively license-expression Signed-off-by: Philippe Ombredanne --- src/packagedcode/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index d3fd962a190..47270f9f4f2 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -30,6 +30,8 @@ from commoncode.datautils import String from commoncode.fileutils import as_posixpath from commoncode.resource import Resource +from license_expression import combine_expressions +from license_expression import Licensing try: from typecode import contenttype From ff4753543c412063111d3139edaef9cbae3fd1d0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 16:27:29 +0200 Subject: [PATCH 26/28] Ensure all licensing usage are guarded Guard the usage also when doing plain expression hanlding that rely on the SPDX license key data from the LicenseIndex/License cache. Signed-off-by: Philippe Ombredanne --- src/packagedcode/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 38fcb956ac2..133e15d829a 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -1746,6 +1746,15 @@ def update( return True def refresh_license_expressions(self, default_relation='AND'): + """ + Refresh the standard and SPDX license expressions. Do nothing is the licensing code + is not available. + """ + try: + from packagedcode.licensing import get_declared_license_expression_spdx + except ImportError: + return + if self.license_detections: self.declared_license_expression = str(combine_expressions( expressions=[ From 3a04ec525000242ab95ee2ba828c7f22a3a8cdae Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 May 2024 16:37:58 +0200 Subject: [PATCH 27/28] Update changelog for packagedcode Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 46b6badfa94..6cf6d3d940f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,7 +40,11 @@ v33.0.0 (next next, roadmap) of these in other summary plugins. See https://github.com/nexB/scancode-toolkit/issues/1745 -v32.1.0 (next, roadmap) +- The Python API of the packagedcode module has changed. The package datafile "handlers" registry + that used to be in packagedcode/__init__.py has been moved to packagedcode/handlers.py + + +v32.1.0 2024-03-22 ---------------------------- New CLI options: From 67b7ef75c9fa5d81c52dcd7777dc703b03301e6c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 9 May 2024 09:17:48 +0200 Subject: [PATCH 28/28] Document vendored flot Signed-off-by: Philippe Ombredanne --- .../flot-0.7.2-py3-none-any.whl.ABOUT | 10 +++- etc/thirdparty/flot.LICENSE | 52 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 etc/thirdparty/flot.LICENSE diff --git a/etc/thirdparty/flot-0.7.2-py3-none-any.whl.ABOUT b/etc/thirdparty/flot-0.7.2-py3-none-any.whl.ABOUT index 08bec68fa4f..64c8a6063ab 100644 --- a/etc/thirdparty/flot-0.7.2-py3-none-any.whl.ABOUT +++ b/etc/thirdparty/flot-0.7.2-py3-none-any.whl.ABOUT @@ -1,2 +1,10 @@ -date: 2024-04-26 +about_resource: flot-0.7.2-py3-none-any.whl +name: flot +version: 0.7.2 download_url: https://files.pythonhosted.org/packages/54/98/9fc7924de9ebff67602f6934fff7411d089468d5aee878d7fd2f9d8be1f5/flot-0.7.2-py3-none-any.whl +copyright: | + Copyright (c) nexB Inc. and contributors + Copyright (c) 2015, Thomas Kluyver and contributors + Copyright (c) Donald Stufft and individual contributors. +license_expression_spdx: BSD-2-clause AND BSD-3-clause +license_file: flot.LICENSE \ No newline at end of file diff --git a/etc/thirdparty/flot.LICENSE b/etc/thirdparty/flot.LICENSE new file mode 100644 index 00000000000..3d8c662dced --- /dev/null +++ b/etc/thirdparty/flot.LICENSE @@ -0,0 +1,52 @@ +Copyright (c) nexB Inc. and contributors +Copyright (c) 2015, Thomas Kluyver and contributors +Copyright (c) Donald Stufft and individual contributors. + +SPDX-License-Identifier: BSD-2-clause AND BSD-3-clause + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file