Support conan package indexing

minecode/tests/test_conan.py

@@ -0,0 +1,108 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import os
+
+import saneyaml
+from django.test import TestCase
+from mock import patch
+from packageurl import PackageURL
+
+import packagedb
+from minecode.utils_test import JsonBasedTesting
+from minecode.visitors import conan
+
+
+class ConanPriorityQueueTests(JsonBasedTesting, TestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), "testfiles")
+
+    def setUp(self):
+        super(ConanPriorityQueueTests, self).setUp()
+        self.package_url1 = PackageURL.from_string("pkg:conan/[email protected]")
+        zlib_conanfile_loc = self.get_test_loc("conan/zlib/manifest/conanfile.py")
+        zlib_conandata_loc = self.get_test_loc("conan/zlib/manifest/conandata.yml")
+        zlib_config_loc = self.get_test_loc("conan/zlib/manifest/config.yml")
+
+        with open(zlib_conanfile_loc) as f:
+            self.zlib_conanfile_contents = f.read()
+
+        with open(zlib_config_loc) as f:
+            self.zlib_config_contents = f.read()
+
+        with open(zlib_conandata_loc) as f:
+            self.zlib_conandata_contents = f.read()
+
+        self.zlib_conandata_contents_dict = saneyaml.load(self.zlib_conandata_contents)
+
+    @patch("requests.get")
+    def test_get_conan_recipe(self, mock_get):
+        mock_get.side_effect = [
+            type(
+                "Response",
+                (),
+                {
+                    "content": self.zlib_config_contents.encode(),
+                    "raise_for_status": lambda: None,
+                },
+            ),
+            type(
+                "Response",
+                (),
+                {
+                    "content": self.zlib_conandata_contents.encode(),
+                    "raise_for_status": lambda: None,
+                },
+            ),
+            type(
+                "Response",
+                (),
+                {
+                    "text": self.zlib_conanfile_contents,
+                    "raise_for_status": lambda: None,
+                },
+            ),
+        ]
+        result_conanfile, result_conandata = conan.get_conan_recipe(
+            self.package_url1.name, self.package_url1.version
+        )
+
+        self.assertEqual(result_conanfile, self.zlib_conanfile_contents)
+        self.assertEqual(result_conandata, self.zlib_conandata_contents_dict)
+
+    def test_get_download_info(self):
+        result_download_url, result_sha256 = conan.get_download_info(
+            self.zlib_conandata_contents_dict, self.package_url1.version
+        )
+        expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+        expected_zlib_sha256 = (
+            "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
+        )
+
+        self.assertEqual(result_download_url, expected_zlib_download_url)
+        self.assertEqual(result_sha256, expected_zlib_sha256)
+
+    @patch("minecode.visitors.conan.get_conan_recipe")
+    def test_map_conan_package(self, mock_get_conan_recipe):
+        mock_get_conan_recipe.return_value = (
+            self.zlib_conanfile_contents,
+            self.zlib_conandata_contents_dict,
+        )
+
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(package_count, 0)
+
+        conan.map_conan_package(self.package_url1)
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(package_count, 1)
+        package = packagedb.models.Package.objects.all().first()
+        expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+
+        self.assertEqual(package.purl, str(self.package_url1))
+        self.assertEqual(package.download_url, expected_zlib_download_url)

minecode/tests/testfiles/conan/zlib/manifest.ABOUT

@@ -0,0 +1,9 @@
+about_resource: manifest/
+name: conan-center-index
+version: 30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc
+download_url: https://github.com/conan-io/conan-center-index/tree/30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc/recipes/zlib/
+vcs_repository: https://github.com/conan-io/conan-center-index.git
+license: mit
+license_text: manifest.LICENSE
+description: Recipes for the ConanCenter repository.
+notes: recipe used for testing conan package indexing 

minecode/tests/testfiles/conan/zlib/manifest.LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Conan.io
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

minecode/tests/testfiles/conan/zlib/manifest/conandata.yml

@@ -0,0 +1,57 @@
+sources:
+  "1.3.1":
+    url:
+      - "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.1.tar.gz"
+    sha256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
+  "1.3":
+    url:
+      - "https://zlib.net/fossils/zlib-1.3.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz"
+    sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e"
+  "1.2.13":
+    url:
+      - "https://zlib.net/fossils/zlib-1.2.13.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz"
+    sha256: "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30"
+  "1.2.12":
+    url: "https://zlib.net/fossils/zlib-1.2.12.tar.gz"
+    sha256: "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
+  "1.2.11":
+    url: "https://zlib.net/fossils/zlib-1.2.11.tar.gz"
+    sha256: "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1"
+patches:
+  "1.3.1":
+    - patch_file: "patches/1.3.1/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix"
+      patch_type: "conan"
+  "1.3":
+    - patch_file: "patches/1.3/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+  "1.2.13":
+    - patch_file: "patches/1.2.13/0001-Fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+  "1.2.12":
+    - patch_file: "patches/1.2.x/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+    - patch_file: "patches/1.2.x/0004-Fix-a-bug-when-getting-a-gzip-header-extra-field-wit.patch"
+      patch_description: "CVE-2022-37434: Fix a bug when getting a gzip header extra field with inflate()"
+      patch_type: "vulnerability"
+      patch_source: "https://github.com/madler/zlib/commit/eff308af425b67093bab25f80f1ae950166bece1"
+      sha256: "15e3c177dc2a034a22e02490a97ba5b1719aae3f8129a06c16d727b661d1650f"
+    - patch_file: "patches/1.2.x/0005-Fix-extra-field-processing-bug-that-dereferences-NUL.patch"
+      patch_description: "CVE-2022-37434: Fix extra field processing bug that dereferences NULL state->head"
+      patch_type: "vulnerability"
+      patch_source: "https://github.com/madler/zlib/commit/1eb7682f845ac9e9bf9ae35bbfb3bad5dacbd91d"
+      sha256: "cdd69eb3251728b1875c8ecae6427b50aa750b4045ef984ab79b6c07b7e6dd3a"
+  "1.2.11":
+    - patch_file: "patches/1.2.x/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+    - patch_file: "patches/1.2.x/0003-gzguts-fix-widechar-condition.patch"
+      patch_description: "fix condition for WIDECHAR usage"
+      patch_type: "portability"
+      patch_source: "https://github.com/madler/zlib/issues/268"

minecode/tests/testfiles/conan/zlib/manifest/conanfile.py

@@ -0,0 +1,107 @@
+class ConanFile:
+    pass
+
+required_conan_version = ">=1.53.0"
+
+
+class ZlibConan(ConanFile):
+    name = "zlib"
+    package_type = "library"
+    url = "https://github.com/conan-io/conan-center-index"
+    homepage = "https://zlib.net"
+    license = "Zlib"
+    description = ("A Massively Spiffy Yet Delicately Unobtrusive Compression Library "
+                   "(Also Free, Not to Mention Unencumbered by Patents)")
+    topics = ("zlib", "compression")
+
+    settings = "os", "arch", "compiler", "build_type"
+    options = {
+        "shared": [True, False],
+        "fPIC": [True, False],
+    }
+    default_options = {
+        "shared": False,
+        "fPIC": True,
+    }
+
+    @property
+    def _is_mingw(self):
+        return self.settings.os == "Windows" and self.settings.compiler == "gcc"
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            del self.options.fPIC
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+        self.settings.rm_safe("compiler.libcxx")
+        self.settings.rm_safe("compiler.cppstd")
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version],
+            destination=self.source_folder, strip_root=True)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.variables["SKIP_INSTALL_ALL"] = False
+        tc.variables["SKIP_INSTALL_LIBRARIES"] = False
+        tc.variables["SKIP_INSTALL_HEADERS"] = False
+        tc.variables["SKIP_INSTALL_FILES"] = True
+        # Correct for misuse of "${CMAKE_INSTALL_PREFIX}/" in CMakeLists.txt
+        tc.variables["INSTALL_LIB_DIR"] = "lib"
+        tc.variables["INSTALL_INC_DIR"] = "include"
+        tc.variables["ZLIB_BUILD_EXAMPLES"] = False
+        tc.generate()
+
+    def _patch_sources(self):
+        apply_conandata_patches(self)
+
+        is_apple_clang12 = self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version) >= "12.0"
+        if not is_apple_clang12:
+            for filename in ['zconf.h', 'zconf.h.cmakein', 'zconf.h.in']:
+                filepath = os.path.join(self.source_folder, filename)
+                replace_in_file(self, filepath,
+                                      '#ifdef HAVE_UNISTD_H    '
+                                      '/* may be set to #if 1 by ./configure */',
+                                      '#if defined(HAVE_UNISTD_H) && (1-HAVE_UNISTD_H-1 != 0)')
+                replace_in_file(self, filepath,
+                                      '#ifdef HAVE_STDARG_H    '
+                                      '/* may be set to #if 1 by ./configure */',
+                                      '#if defined(HAVE_STDARG_H) && (1-HAVE_STDARG_H-1 != 0)')
+
+    def build(self):
+        self._patch_sources()
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def _extract_license(self):
+        tmp = load(self, os.path.join(self.source_folder, "zlib.h"))
+        license_contents = tmp[2:tmp.find("*/", 1)]
+        return license_contents
+
+    def package(self):
+        save(self, os.path.join(self.package_folder, "licenses", "LICENSE"), self._extract_license())
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.set_property("cmake_find_mode", "both")
+        self.cpp_info.set_property("cmake_file_name", "ZLIB")
+        self.cpp_info.set_property("cmake_target_name", "ZLIB::ZLIB")
+        self.cpp_info.set_property("pkg_config_name", "zlib")
+        if self.settings.os == "Windows" and not self._is_mingw:
+            libname = "zdll" if self.options.shared else "zlib"
+        else:
+            libname = "z"
+        self.cpp_info.libs = [libname]
+
+        self.cpp_info.names["cmake_find_package"] = "ZLIB"
+        self.cpp_info.names["cmake_find_package_multi"] = "ZLIB"

minecode/tests/testfiles/conan/zlib/manifest/config.yml

@@ -0,0 +1,11 @@
+versions:
+  "1.3.1":
+    folder: all
+  "1.3":
+    folder: all
+  "1.2.13":
+    folder: all
+  "1.2.12":
+    folder: all
+  "1.2.11":
+    folder: all