From 126035ee63a9f2b1f59697e88aa4c184f44c3927 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 13 Feb 2024 22:41:18 +0530 Subject: [PATCH 1/3] Support conan package indexing Signed-off-by: Keshav Priyadarshi --- minecode/visitors/conan.py | 159 +++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 minecode/visitors/conan.py diff --git a/minecode/visitors/conan.py b/minecode/visitors/conan.py new file mode 100644 index 00000000..15ec2678 --- /dev/null +++ b/minecode/visitors/conan.py @@ -0,0 +1,159 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import logging + +import requests +import saneyaml +from packagedcode.conan import ConanFileHandler +from packageurl import PackageURL + +from minecode import priority_router +from packagedb.models import PackageContentType + +""" +Collect Conan packages from Conan Central. +""" + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def get_yaml_response(url): + """ + Fetch YAML content from the url and return it as a dictionary. + """ + try: + response = requests.get(url) + response.raise_for_status() + content = response.content.decode("utf-8") + return saneyaml.load(content) + except requests.exceptions.HTTPError as err: + logger.error(f"HTTP error occurred: {err}") + + +def get_conan_recipe(name, version): + """ + Return the contents of the `conanfile.py` and `conandata.yml` file for + the conan package described by name and version string. + """ + base_index_url = ( + "https://raw.githubusercontent.com/conan-io/" + "conan-center-index/master/recipes/" + ) + + conan_central_config_url = f"{base_index_url}/{name}/config.yml" + config = get_yaml_response(conan_central_config_url) + if not config: + return None, None + + versions = config.get("versions", {}) + recipe_location = versions.get(version, {}) + folder = recipe_location.get("folder") + + folder = recipe_location.get("folder") + if not folder: + logger.error(f"No folder found for version {version} of package {name}") + return None, None + + conanfile_py_url = f"{base_index_url}/{name}/{folder}/conanfile.py" + conandata_yml_url = f"{base_index_url}/{name}/{folder}/conandata.yml" + + conandata = get_yaml_response(conandata_yml_url) + + try: + response = requests.get(conanfile_py_url) + response.raise_for_status() + conanfile = response.text + except requests.exceptions.HTTPError as err: + logger.error( + f"HTTP error occurred while fetching conanfile.py for {name} {version}: {err}" + ) + conanfile = None + + return conanfile, conandata + + +def get_download_info(conandata, version): + """ + Return download_url and SHA256 hash from `conandata.yml`. + """ + sources = conandata.get("sources", {}) + pkg_data = sources.get(version, {}) + + download_url = pkg_data.get("url") + sha256 = pkg_data.get("sha256") + + if isinstance(download_url, list): + download_url = download_url[0] + + return download_url, sha256 + + +def map_conan_package(package_url): + """ + Add a conan `package_url` to the PackageDB. + + Return an error string if any errors are encountered during the process + """ + from minecode.model_utils import add_package_to_scan_queue + from minecode.model_utils import merge_or_create_package + + conanfile, conandata = get_conan_recipe( + name=package_url.name, + version=package_url.version, + ) + + download_url, sha256 = get_download_info(conandata, package_url.version) + + if not conanfile: + error = f"Package does not exist on conan central: {package_url}" + logger.error(error) + return error + if not download_url: + error = f"Package download_url does not exist on conan central: {package_url}" + logger.error(error) + return error + + package = ConanFileHandler._parse(conan_recipe=conanfile) + package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE + package.version = package_url.version + package.download_url = download_url + package.sha256 = sha256 + + db_package, _, _, error = merge_or_create_package(package, visit_level=0) + + # Submit package for scanning + if db_package: + add_package_to_scan_queue(db_package) + + return error + + +@priority_router.route("pkg:conan/.*") +def process_request(purl_str): + """ + Process `priority_resource_uri` containing a conan Package URL (PURL) as a + URI. + + This involves obtaining Package information for the PURL from + https://github.com/conan-io/conan-center-index and using it to create a new + PackageDB entry. The package is then added to the scan queue afterwards. + """ + package_url = PackageURL.from_string(purl_str) + if not package_url.version: + return + + error_msg = map_conan_package(package_url) + + if error_msg: + return error_msg From ba5000764e65dbba2c8a4fe7862daf131cf7030a Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 15 Feb 2024 23:11:45 +0530 Subject: [PATCH 2/3] Test conan package indexing Signed-off-by: Keshav Priyadarshi --- minecode/tests/test_conan.py | 108 ++++++++++++++++++ .../tests/testfiles/conan/zlib/manifest.ABOUT | 9 ++ .../testfiles/conan/zlib/manifest.LICENSE | 21 ++++ .../conan/zlib/manifest/conandata.yml | 57 +++++++++ .../conan/zlib/manifest/conanfile.py | 104 +++++++++++++++++ .../testfiles/conan/zlib/manifest/config.yml | 11 ++ 6 files changed, 310 insertions(+) create mode 100644 minecode/tests/test_conan.py create mode 100644 minecode/tests/testfiles/conan/zlib/manifest.ABOUT create mode 100644 minecode/tests/testfiles/conan/zlib/manifest.LICENSE create mode 100644 minecode/tests/testfiles/conan/zlib/manifest/conandata.yml create mode 100644 minecode/tests/testfiles/conan/zlib/manifest/conanfile.py create mode 100644 minecode/tests/testfiles/conan/zlib/manifest/config.yml diff --git a/minecode/tests/test_conan.py b/minecode/tests/test_conan.py new file mode 100644 index 00000000..f15b9611 --- /dev/null +++ b/minecode/tests/test_conan.py @@ -0,0 +1,108 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import os + +import saneyaml +from django.test import TestCase +from mock import patch +from packageurl import PackageURL + +import packagedb +from minecode.utils_test import JsonBasedTesting +from minecode.visitors import conan + + +class ConanPriorityQueueTests(JsonBasedTesting, TestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), "testfiles") + + def setUp(self): + super(ConanPriorityQueueTests, self).setUp() + self.package_url1 = PackageURL.from_string("pkg:conan/zlib@1.3.1") + zlib_conanfile_loc = self.get_test_loc("conan/zlib/manifest/conanfile.py") + zlib_conandata_loc = self.get_test_loc("conan/zlib/manifest/conandata.yml") + zlib_config_loc = self.get_test_loc("conan/zlib/manifest/config.yml") + + with open(zlib_conanfile_loc) as f: + self.zlib_conanfile_contents = f.read() + + with open(zlib_config_loc) as f: + self.zlib_config_contents = f.read() + + with open(zlib_conandata_loc) as f: + self.zlib_conandata_contents = f.read() + + self.zlib_conandata_contents_dict = saneyaml.load(self.zlib_conandata_contents) + + @patch("requests.get") + def test_get_conan_recipe(self, mock_get): + mock_get.side_effect = [ + type( + "Response", + (), + { + "content": self.zlib_config_contents.encode(), + "raise_for_status": lambda: None, + }, + ), + type( + "Response", + (), + { + "content": self.zlib_conandata_contents.encode(), + "raise_for_status": lambda: None, + }, + ), + type( + "Response", + (), + { + "text": self.zlib_conanfile_contents, + "raise_for_status": lambda: None, + }, + ), + ] + result_conanfile, result_conandata = conan.get_conan_recipe( + self.package_url1.name, self.package_url1.version + ) + + self.assertEqual(result_conanfile, self.zlib_conanfile_contents) + self.assertEqual(result_conandata, self.zlib_conandata_contents_dict) + + def test_get_download_info(self): + result_download_url, result_sha256 = conan.get_download_info( + self.zlib_conandata_contents_dict, self.package_url1.version + ) + expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz" + expected_zlib_sha256 = ( + "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + ) + + self.assertEqual(result_download_url, expected_zlib_download_url) + self.assertEqual(result_sha256, expected_zlib_sha256) + + @patch("minecode.visitors.conan.get_conan_recipe") + def test_map_conan_package(self, mock_get_conan_recipe): + mock_get_conan_recipe.return_value = ( + self.zlib_conanfile_contents, + self.zlib_conandata_contents_dict, + ) + + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(package_count, 0) + + conan.map_conan_package(self.package_url1) + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(package_count, 1) + package = packagedb.models.Package.objects.all().first() + expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz" + + self.assertEqual(package.purl, str(self.package_url1)) + self.assertEqual(package.download_url, expected_zlib_download_url) diff --git a/minecode/tests/testfiles/conan/zlib/manifest.ABOUT b/minecode/tests/testfiles/conan/zlib/manifest.ABOUT new file mode 100644 index 00000000..f5615c81 --- /dev/null +++ b/minecode/tests/testfiles/conan/zlib/manifest.ABOUT @@ -0,0 +1,9 @@ +about_resource: manifest/ +name: conan-center-index +version: 30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc +download_url: https://github.com/conan-io/conan-center-index/tree/30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc/recipes/zlib/ +vcs_repository: https://github.com/conan-io/conan-center-index.git +license: mit +license_text: manifest.LICENSE +description: Recipes for the ConanCenter repository. +notes: recipe used for testing conan package indexing \ No newline at end of file diff --git a/minecode/tests/testfiles/conan/zlib/manifest.LICENSE b/minecode/tests/testfiles/conan/zlib/manifest.LICENSE new file mode 100644 index 00000000..2b6d3822 --- /dev/null +++ b/minecode/tests/testfiles/conan/zlib/manifest.LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml b/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml new file mode 100644 index 00000000..5010531f --- /dev/null +++ b/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml @@ -0,0 +1,57 @@ +sources: + "1.3.1": + url: + - "https://zlib.net/fossils/zlib-1.3.1.tar.gz" + - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.1.tar.gz" + sha256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + "1.3": + url: + - "https://zlib.net/fossils/zlib-1.3.tar.gz" + - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz" + sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + "1.2.13": + url: + - "https://zlib.net/fossils/zlib-1.2.13.tar.gz" + - "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz" + sha256: "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30" + "1.2.12": + url: "https://zlib.net/fossils/zlib-1.2.12.tar.gz" + sha256: "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9" + "1.2.11": + url: "https://zlib.net/fossils/zlib-1.2.11.tar.gz" + sha256: "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1" +patches: + "1.3.1": + - patch_file: "patches/1.3.1/0001-fix-cmake.patch" + patch_description: "separate static/shared builds, disable debug suffix" + patch_type: "conan" + "1.3": + - patch_file: "patches/1.3/0001-fix-cmake.patch" + patch_description: "separate static/shared builds, disable debug suffix, disable building examples" + patch_type: "conan" + "1.2.13": + - patch_file: "patches/1.2.13/0001-Fix-cmake.patch" + patch_description: "separate static/shared builds, disable debug suffix, disable building examples" + patch_type: "conan" + "1.2.12": + - patch_file: "patches/1.2.x/0001-fix-cmake.patch" + patch_description: "separate static/shared builds, disable debug suffix, disable building examples" + patch_type: "conan" + - patch_file: "patches/1.2.x/0004-Fix-a-bug-when-getting-a-gzip-header-extra-field-wit.patch" + patch_description: "CVE-2022-37434: Fix a bug when getting a gzip header extra field with inflate()" + patch_type: "vulnerability" + patch_source: "https://github.com/madler/zlib/commit/eff308af425b67093bab25f80f1ae950166bece1" + sha256: "15e3c177dc2a034a22e02490a97ba5b1719aae3f8129a06c16d727b661d1650f" + - patch_file: "patches/1.2.x/0005-Fix-extra-field-processing-bug-that-dereferences-NUL.patch" + patch_description: "CVE-2022-37434: Fix extra field processing bug that dereferences NULL state->head" + patch_type: "vulnerability" + patch_source: "https://github.com/madler/zlib/commit/1eb7682f845ac9e9bf9ae35bbfb3bad5dacbd91d" + sha256: "cdd69eb3251728b1875c8ecae6427b50aa750b4045ef984ab79b6c07b7e6dd3a" + "1.2.11": + - patch_file: "patches/1.2.x/0001-fix-cmake.patch" + patch_description: "separate static/shared builds, disable debug suffix, disable building examples" + patch_type: "conan" + - patch_file: "patches/1.2.x/0003-gzguts-fix-widechar-condition.patch" + patch_description: "fix condition for WIDECHAR usage" + patch_type: "portability" + patch_source: "https://github.com/madler/zlib/issues/268" \ No newline at end of file diff --git a/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py new file mode 100644 index 00000000..ac21a024 --- /dev/null +++ b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py @@ -0,0 +1,104 @@ +required_conan_version = ">=1.53.0" + + +class ZlibConan(ConanFile): + name = "zlib" + package_type = "library" + url = "https://github.com/conan-io/conan-center-index" + homepage = "https://zlib.net" + license = "Zlib" + description = ("A Massively Spiffy Yet Delicately Unobtrusive Compression Library " + "(Also Free, Not to Mention Unencumbered by Patents)") + topics = ("zlib", "compression") + + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + } + + @property + def _is_mingw(self): + return self.settings.os == "Windows" and self.settings.compiler == "gcc" + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + self.settings.rm_safe("compiler.libcxx") + self.settings.rm_safe("compiler.cppstd") + + def layout(self): + cmake_layout(self, src_folder="src") + + def source(self): + get(self, **self.conan_data["sources"][self.version], + destination=self.source_folder, strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["SKIP_INSTALL_ALL"] = False + tc.variables["SKIP_INSTALL_LIBRARIES"] = False + tc.variables["SKIP_INSTALL_HEADERS"] = False + tc.variables["SKIP_INSTALL_FILES"] = True + # Correct for misuse of "${CMAKE_INSTALL_PREFIX}/" in CMakeLists.txt + tc.variables["INSTALL_LIB_DIR"] = "lib" + tc.variables["INSTALL_INC_DIR"] = "include" + tc.variables["ZLIB_BUILD_EXAMPLES"] = False + tc.generate() + + def _patch_sources(self): + apply_conandata_patches(self) + + is_apple_clang12 = self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version) >= "12.0" + if not is_apple_clang12: + for filename in ['zconf.h', 'zconf.h.cmakein', 'zconf.h.in']: + filepath = os.path.join(self.source_folder, filename) + replace_in_file(self, filepath, + '#ifdef HAVE_UNISTD_H ' + '/* may be set to #if 1 by ./configure */', + '#if defined(HAVE_UNISTD_H) && (1-HAVE_UNISTD_H-1 != 0)') + replace_in_file(self, filepath, + '#ifdef HAVE_STDARG_H ' + '/* may be set to #if 1 by ./configure */', + '#if defined(HAVE_STDARG_H) && (1-HAVE_STDARG_H-1 != 0)') + + def build(self): + self._patch_sources() + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _extract_license(self): + tmp = load(self, os.path.join(self.source_folder, "zlib.h")) + license_contents = tmp[2:tmp.find("*/", 1)] + return license_contents + + def package(self): + save(self, os.path.join(self.package_folder, "licenses", "LICENSE"), self._extract_license()) + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.set_property("cmake_find_mode", "both") + self.cpp_info.set_property("cmake_file_name", "ZLIB") + self.cpp_info.set_property("cmake_target_name", "ZLIB::ZLIB") + self.cpp_info.set_property("pkg_config_name", "zlib") + if self.settings.os == "Windows" and not self._is_mingw: + libname = "zdll" if self.options.shared else "zlib" + else: + libname = "z" + self.cpp_info.libs = [libname] + + self.cpp_info.names["cmake_find_package"] = "ZLIB" + self.cpp_info.names["cmake_find_package_multi"] = "ZLIB" \ No newline at end of file diff --git a/minecode/tests/testfiles/conan/zlib/manifest/config.yml b/minecode/tests/testfiles/conan/zlib/manifest/config.yml new file mode 100644 index 00000000..e42c3dfb --- /dev/null +++ b/minecode/tests/testfiles/conan/zlib/manifest/config.yml @@ -0,0 +1,11 @@ +versions: + "1.3.1": + folder: all + "1.3": + folder: all + "1.2.13": + folder: all + "1.2.12": + folder: all + "1.2.11": + folder: all \ No newline at end of file From e070d95efab63d6346ff2eef033946b9808fa8e9 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 29 Mar 2024 00:18:58 +0530 Subject: [PATCH 3/3] Fix conan test data Signed-off-by: Keshav Priyadarshi --- minecode/tests/testfiles/conan/zlib/manifest/conanfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py index ac21a024..72f2e5fc 100644 --- a/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py +++ b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py @@ -1,3 +1,6 @@ +class ConanFile: + pass + required_conan_version = ">=1.53.0"