diff --git a/.env b/.env new file mode 100644 index 00000000..18990d37 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +# Used for VSCode. +# See https://code.visualstudio.com/docs/python/environments. +PYTHONPATH=python diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 014eed6e..6855bdac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,13 @@ jobs: --toolchain=llvm12 --expected-major-compiler-version=12 + - name: centos7-x86_64-clang13 + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_centos7:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13 + --expected-major-compiler-version=13 + # --------------------------------------------------------------------------------------- # Ubuntu 18.04 # --------------------------------------------------------------------------------------- @@ -118,6 +125,8 @@ jobs: --devtoolset=9 --expected-major-compiler-version=9 + # Clang/LLVM 12 + - name: almalinux8-x86_64-clang12 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 @@ -164,6 +173,53 @@ jobs: --expected-major-compiler-version=12 --lto=full + # Clang/LLVM 13 + - name: almalinux8-x86_64-clang13 + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13 + --expected-major-compiler-version=13 + + - name: almalinux8-x86_64-clang13-thin-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13 + --expected-major-compiler-version=13 + --lto=thin + + - name: almalinux8-x86_64-clang13-full-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13 + --expected-major-compiler-version=13 + --lto=full + + - name: almalinux8-x86_64-clang13-linuxbrew + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13_linuxbrew + --expected-major-compiler-version=13 + + - name: almalinux8-x86_64-clang13-linuxbrew-thin-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13_linuxbrew + --expected-major-compiler-version=13 + --lto=thin + + - name: almalinux8-x86_64-clang13-linuxbrew-full-lto + os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8:v2021-08-27T03_10_19 + build_thirdparty_args: >- + --toolchain=llvm13_linuxbrew + --expected-major-compiler-version=13 + --lto=full + # --------------------------------------------------------------------------------------- # macOS # --------------------------------------------------------------------------------------- diff --git a/python/build_definitions/llvm1x_libcxx.py b/python/build_definitions/llvm1x_libcxx.py index ec9bd599..ac643562 100644 --- a/python/build_definitions/llvm1x_libcxx.py +++ b/python/build_definitions/llvm1x_libcxx.py @@ -90,11 +90,12 @@ def get_source_subdir_name(self) -> str: def get_additional_cmake_args(self, builder: BuilderInterface) -> List[str]: llvm_src_path = builder.fs_layout.get_source_path(self) - return [ + args = [ '-DLIBCXXABI_LIBCXX_PATH=%s' % os.path.join(llvm_src_path, 'libcxx'), '-DLIBCXXABI_USE_COMPILER_RT=ON', '-DLIBCXXABI_USE_LLVM_UNWINDER=ON', ] + return args def build(self, builder: BuilderInterface) -> None: super().build(builder) @@ -125,3 +126,46 @@ def get_additional_cmake_args(self, builder: BuilderInterface) -> List[str]: '-DLIBCXX_CXX_ABI=libcxxabi', '-DLIBCXXABI_USE_LLVM_UNWINDER=ON', ] + + +class LibCxxWithAbiDependency(Llvm1xLibCxxDependencyBase): + """ + A combined dependency for libc++ and libc++abi. + + Using the approach described at: + + https://libcxx.llvm.org/BuildingLibcxx.html + + Based on the following instructions: + + $ git clone https://github.com/llvm/llvm-project.git + $ cd llvm-project + $ mkdir build + $ cmake -G Ninja -S runtimes -B build -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" + $ ninja -C build cxx cxxabi unwind + $ ninja -C build check-cxx check-cxxabi check-unwind + $ ninja -C build install-cxx install-cxxabi install-unwind + + Using this with LLVM/Clang 13 or later. + """ + + def __init__(self, version: str) -> None: + super(LibCxxWithAbiDependency, self).__init__( + name='llvm1x_libcxx_with_abi', + version=version) + + def get_source_subdir_name(self) -> str: + return 'runtimes' + + def get_additional_cmake_args(self, builder: BuilderInterface) -> List[str]: + return [ + '-DLLVM_ENABLE_RUNTIMES=libcxx;libcxxabi', + ] + + def get_compiler_wrapper_ld_flags_to_append(self, builder: 'BuilderInterface') -> List[str]: + extra_ld_flags = super().get_compiler_wrapper_ld_flags_to_append(builder) + if builder.build_type == BUILD_TYPE_TSAN: + # It is not clear why in Clang 13 this suddenly becomes necessary in order to avoid + # failing with undefined TSAN-related symbols while linking shared libraries. + extra_ld_flags.append('-Wl,--unresolved-symbols=ignore-all') + return extra_ld_flags diff --git a/python/build_definitions/llvm_runtimes.py b/python/build_definitions/llvm_runtimes.py new file mode 100644 index 00000000..e69de29b diff --git a/python/yugabyte_db_thirdparty/builder.py b/python/yugabyte_db_thirdparty/builder.py index 2d9d8cfd..d63c88f5 100644 --- a/python/yugabyte_db_thirdparty/builder.py +++ b/python/yugabyte_db_thirdparty/builder.py @@ -35,7 +35,7 @@ get_build_def_module, ) from yugabyte_db_thirdparty.builder_helpers import PLACEHOLDER_RPATH, get_make_parallelism, \ - get_rpath_flag, sanitize_flags_line_for_log, log_and_set_env_var_to_list + get_rpath_flag, log_and_set_env_var_to_list, format_cmake_args_for_log from yugabyte_db_thirdparty.builder_helpers import is_ninja_available from yugabyte_db_thirdparty.builder_interface import BuilderInterface from yugabyte_db_thirdparty.cmd_line_args import parse_cmd_line_args @@ -74,6 +74,8 @@ ) from yugabyte_db_thirdparty.macos import get_min_supported_macos_version from yugabyte_db_thirdparty.linuxbrew import get_linuxbrew_dir, using_linuxbrew, set_linuxbrew_dir +from yugabyte_db_thirdparty.constants import COMPILER_WRAPPER_LD_FLAGS_TO_APPEND_ENV_VAR_NAME + ASAN_FLAGS = [ '-fsanitize=address', @@ -191,11 +193,6 @@ def parse_args(self) -> None: compiler_prefix = self.toolchain.toolchain_root single_compiler_type = self.toolchain.get_compiler_type() self.toolchain.write_url_and_path_files() - if single_compiler_type == 'clang' and using_linuxbrew(): - log("Automatically enabling compiler wrapper for a Clang Linuxbrew-targeting build " - "and configuring it to disallow headers from /usr/include.") - self.args.use_compiler_wrapper = True - os.environ['YB_DISALLOWED_INCLUDE_DIRS'] = '/usr/include' else: compiler_prefix = self.args.compiler_prefix single_compiler_type = self.args.single_compiler_type @@ -209,6 +206,17 @@ def parse_args(self) -> None: use_ccache=self.args.use_ccache, expected_major_compiler_version=self.args.expected_major_compiler_version ) + llvm_major_version: Optional[int] = self.compiler_choice.get_llvm_major_version() + if llvm_major_version: + if using_linuxbrew(): + log("Automatically enabling compiler wrapper for a Clang Linuxbrew-targeting build") + log("Disallowing the use of headers in /usr/include") + os.environ['YB_DISALLOWED_INCLUDE_DIRS'] = '/usr/include' + self.args.use_compiler_wrapper = True + if llvm_major_version >= 13: + log("Automatically enabling compiler wrapper for Clang major version 13 or higher") + self.args.use_compiler_wrapper = True + self.compiler_choice.use_compiler_wrapper = self.args.use_compiler_wrapper self.lto_type = self.args.lto @@ -258,18 +266,23 @@ def populate_dependencies(self) -> None: llvm_version_str = self.toolchain.get_llvm_version_str() else: llvm_version_str = self.compiler_choice.get_llvm_version_str() - self.dependencies += [ - # New LLVM. We will keep supporting new LLVM versions here. + + self.dependencies.append( get_build_def_module('llvm1x_libunwind').Llvm1xLibUnwindDependency( version=llvm_version_str - ), - get_build_def_module('llvm1x_libcxx').Llvm1xLibCxxAbiDependency( - version=llvm_version_str - ), - get_build_def_module('llvm1x_libcxx').Llvm1xLibCxxDependency( - version=llvm_version_str - ), - ] + )) + libcxx_dep_module = get_build_def_module('llvm1x_libcxx') + if llvm_major_version >= 13: + self.dependencies.append( + libcxx_dep_module.LibCxxWithAbiDependency(version=llvm_version_str)) + else: + # It is important that we build libc++abi first, and only then build libc++. + self.dependencies += [ + libcxx_dep_module.Llvm1xLibCxxAbiDependency(version=llvm_version_str), + libcxx_dep_module.Llvm1xLibCxxDependency(version=llvm_version_str), + ] + self.additional_allowed_shared_lib_paths.add( + get_clang_library_dir(self.compiler_choice.get_c_compiler())) else: self.dependencies.append(get_build_def_module('libunwind').LibUnwindDependency()) @@ -589,8 +602,7 @@ def build_with_cmake( def do_build_with_cmake(additional_cmake_args: List[str] = []) -> None: final_cmake_args = args + additional_cmake_args log("CMake command line (one argument per line):\n%s" % - "\n".join([(" " * 4 + sanitize_flags_line_for_log(line)) - for line in final_cmake_args])) + format_cmake_args_for_log(final_cmake_args)) cmake_configure_script_path = os.path.abspath('yb_build_with_cmake.sh') build_tool_cmd = [ @@ -786,6 +798,9 @@ def init_linux_clang1x_flags(self, dep: Dependency) -> None: Flags for Clang 10 and beyond. We are using LLVM-supplied libunwind and compiler-rt in this configuration. """ + llvm_major_version = self.compiler_choice.get_llvm_major_version() + assert llvm_major_version is not None + if not using_linuxbrew(): # We don't build compiler-rt for Linuxbrew yet. # TODO: we can build compiler-rt here the same way we build other LLVM components, @@ -823,21 +838,36 @@ def init_linux_clang1x_flags(self, dep: Dependency) -> None: # TODO mbautin: refactor to polymorphism is_libcxxabi = dep.name.endswith('_libcxxabi') is_libcxx = dep.name.endswith('_libcxx') + + is_libcxx_with_abi = dep.name.endswith('_libcxx_with_abi') + log("Dependency name: %s, is_libcxxabi: %s, is_libcxx: %s", dep.name, is_libcxxabi, is_libcxx) if self.build_type == BUILD_TYPE_ASAN: self.compiler_flags.append('-shared-libasan') - if is_libcxxabi: + if is_libcxxabi or is_libcxx_with_abi: # To avoid an infinite loop in UBSAN. # https://monorail-prod.appspot.com/p/chromium/issues/detail?id=609786 # This comment: # https://gist.githubusercontent.com/mbautin/ad9ea4715669da3b3a5fb9495659c4a9/raw self.compiler_flags.append('-fno-sanitize=vptr') + # Unfortunately, for the combined libc++ and libc++abi build in Clang 13 or later, + # we also disable this check in libc++, where in theory it could have been + # enabled. + + # The description of this check from + # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html: + # + # -fsanitize=vptr: Use of an object whose vptr indicates that it is of the wrong + # dynamic type, or that its lifetime has not begun or has ended. Incompatible with + # -fno-rtti. Link must be performed by clang++, not clang, to make sure C++-specific + # parts of the runtime library and C++ standard libraries are present. + assert self.compiler_choice.cc is not None - compiler_rt_lib_dir = get_clang_library_dir(self.compiler_choice.cc) + compiler_rt_lib_dir = get_clang_library_dir(self.compiler_choice.get_c_compiler()) self.add_lib_dir_and_rpath(compiler_rt_lib_dir) ubsan_lib_name = f'clang_rt.ubsan_minimal-{platform.processor()}' ubsan_lib_so_path = os.path.join(compiler_rt_lib_dir, f'lib{ubsan_lib_name}.so') @@ -845,13 +875,16 @@ def init_linux_clang1x_flags(self, dep: Dependency) -> None: raise IOError(f"UBSAN library not found at {ubsan_lib_so_path}") self.ld_flags.append(f'-l{ubsan_lib_name}') + if self.build_type == BUILD_TYPE_TSAN and llvm_major_version >= 13: + self.executable_only_ld_flags.extend(['-fsanitize=thread']) + self.ld_flags += ['-lunwind'] libcxx_installed_include, libcxx_installed_lib = self.get_libcxx_dirs(self.build_type) log("libc++ include directory: %s", libcxx_installed_include) log("libc++ library directory: %s", libcxx_installed_lib) - if not is_libcxx and not is_libcxxabi: + if not is_libcxx and not is_libcxxabi and not is_libcxx_with_abi: log("Adding special compiler/linker flags for Clang 10+ for dependencies other than " "libc++") self.ld_flags += ['-lc++', '-lc++abi'] @@ -871,7 +904,7 @@ def init_linux_clang1x_flags(self, dep: Dependency) -> None: # libc++ build needs to be able to find libc++abi library installed here. self.ld_flags.append('-L%s' % libcxx_installed_lib) - if is_libcxx or is_libcxxabi: + if is_libcxx or is_libcxxabi or is_libcxx_with_abi: log("Adding special linker flags for Clang 10 or newer for libc++ or libc++abi") # libc++abi needs to be able to find libcxx at runtime, even though it can't always find # it at build time because libc++abi is built first. @@ -991,6 +1024,19 @@ def build_dependency(self, dep: Dependency, only_process_flags: bool = False) -> log_and_set_env_var_to_list( env_vars, 'CPPFLAGS', self.get_effective_preprocessor_flags(dep)) + compiler_wrapper_extra_ld_flags = dep.get_compiler_wrapper_ld_flags_to_append(self) + if compiler_wrapper_extra_ld_flags: + if not self.compiler_choice.use_compiler_wrapper: + raise RuntimeError( + "Need to add extra linker arguments in the compiler wrapper, but compiler " + "wrapper is not being used: %s" % compiler_wrapper_extra_ld_flags) + log_and_set_env_var_to_list( + env_vars, COMPILER_WRAPPER_LD_FLAGS_TO_APPEND_ENV_VAR_NAME, + compiler_wrapper_extra_ld_flags) + + for k, v in env_vars.items(): + log("Setting environment variable %s to: %s" % (k, v)) + if self.build_type == BUILD_TYPE_ASAN: # To avoid errors similar to: # https://gist.githubusercontent.com/mbautin/4b8eec566f54bcc35706dcd97cab1a95/raw diff --git a/python/yugabyte_db_thirdparty/builder_helpers.py b/python/yugabyte_db_thirdparty/builder_helpers.py index 8b30d539..fa4acebf 100644 --- a/python/yugabyte_db_thirdparty/builder_helpers.py +++ b/python/yugabyte_db_thirdparty/builder_helpers.py @@ -14,6 +14,8 @@ import multiprocessing import os +import re + from typing import Dict, Optional, List from yugabyte_db_thirdparty.custom_logging import log @@ -24,6 +26,8 @@ "/tmp/making_sure_we_have_enough_room_to_set_rpath_later_{}_end_of_rpath".format('_' * 256)) PLACEHOLDER_RPATH_FOR_LOG = '/tmp/long_placeholder_rpath' +CMAKE_VAR_RE = re.compile(r'^(-D[A-Z_]+)=(.*)$') + def get_make_parallelism() -> int: return int(os.environ.get('YB_MAKE_PARALLELISM', multiprocessing.cpu_count())) @@ -62,3 +66,25 @@ def log_and_set_env_var_to_list( log('Unsetting env var %s', env_var_name) # When used with EnvVarContext, this will cause the environment variable to be unset. env_var_map[env_var_name] = None + + +def format_cmake_args_for_log(args: List[str]) -> str: + lines = [] + for arg in args: + match = CMAKE_VAR_RE.match(arg) + if match: + cmake_var_name = match.group(1) + cmake_var_value = match.group(2) + cmake_var_value_parts = cmake_var_value.split() + if len(cmake_var_value_parts) > 1: + lines.append('%s="%s' % (cmake_var_name, cmake_var_value_parts[0])) + current_indent = ' ' * (len(cmake_var_name) + 2) + for cmake_var_value_part in cmake_var_value_parts[1:-1]: + lines.append(current_indent + cmake_var_value_part) + lines.append('%s%s"' % (current_indent, cmake_var_value_parts[-1])) + continue + + lines.append(arg) + + indent = " " * 4 + return indent + "\n".join([(indent + sanitize_flags_line_for_log(line)) for line in lines]) diff --git a/python/yugabyte_db_thirdparty/clang_util.py b/python/yugabyte_db_thirdparty/clang_util.py index 00d87041..3040f6f7 100644 --- a/python/yugabyte_db_thirdparty/clang_util.py +++ b/python/yugabyte_db_thirdparty/clang_util.py @@ -53,6 +53,9 @@ def get_clang_library_dir(clang_executable_path: str) -> str: def get_clang_include_dir(clang_executable_path: str) -> str: + """ + Returns a directory such as lib/clang/13.0.1/include relative to the LLVM installation path. + """ library_dirs = get_clang_library_dirs(clang_executable_path) for library_dir in library_dirs: include_dir = os.path.join(library_dir, 'include') diff --git a/python/yugabyte_db_thirdparty/cmd_line_args.py b/python/yugabyte_db_thirdparty/cmd_line_args.py index 3be30873..c20cff45 100644 --- a/python/yugabyte_db_thirdparty/cmd_line_args.py +++ b/python/yugabyte_db_thirdparty/cmd_line_args.py @@ -21,6 +21,7 @@ from yugabyte_db_thirdparty.checksums import CHECKSUM_FILE_NAME from yugabyte_db_thirdparty.util import log from yugabyte_db_thirdparty.toolchain import TOOLCHAIN_TYPES +from yugabyte_db_thirdparty.constants import ADD_CHECKSUM_ARG from build_definitions import BUILD_TYPES @@ -42,7 +43,7 @@ def parse_cmd_line_args() -> argparse.Namespace: action='store_true', default=False, help='Clean, including downloads.') - parser.add_argument('--add-checksum', + parser.add_argument(ADD_CHECKSUM_ARG, help='Compute and add unknown checksums to %s' % CHECKSUM_FILE_NAME, action='store_true') parser.add_argument('--skip', diff --git a/python/yugabyte_db_thirdparty/compiler_choice.py b/python/yugabyte_db_thirdparty/compiler_choice.py index 1e3062d8..51d36573 100644 --- a/python/yugabyte_db_thirdparty/compiler_choice.py +++ b/python/yugabyte_db_thirdparty/compiler_choice.py @@ -35,7 +35,7 @@ ) from packaging.version import parse as parse_version -LOWEST_GCC_VERSION_STR = '5.5.0' +LOWEST_GCC_VERSION_STR = '7.0.0' class CompilerChoice: @@ -82,6 +82,10 @@ def __init__( self.compiler_version_str = None self.expected_major_compiler_version = expected_major_compiler_version + if self.single_compiler_type == 'clang': + # This is necessary because we might want to know Clang version before set_compiler + # is called externally. + self.set_compiler(self.single_compiler_type) def detect_clang_version(self) -> None: """ diff --git a/python/yugabyte_db_thirdparty/compiler_wrapper.py b/python/yugabyte_db_thirdparty/compiler_wrapper.py index a246333c..ab6c8d45 100644 --- a/python/yugabyte_db_thirdparty/compiler_wrapper.py +++ b/python/yugabyte_db_thirdparty/compiler_wrapper.py @@ -3,13 +3,12 @@ import sys import os import subprocess -import logging -import json import shlex from typing import List, Dict -from yugabyte_db_thirdparty.util import shlex_join +from yugabyte_db_thirdparty.util import shlex_join, is_shared_library_name +from yugabyte_db_thirdparty.constants import COMPILER_WRAPPER_LD_FLAGS_TO_APPEND_ENV_VAR_NAME class CompilerWrapper: @@ -57,8 +56,7 @@ def run(self) -> None: use_ccache = os.getenv('YB_THIRDPARTY_USE_CCACHE') == '1' - compiler_path_and_args = self._get_compiler_path_and_args() - + cmd_args: List[str] if use_ccache: os.environ['CCACHE_COMPILER'] = self.real_compiler_path cmd_args = ['ccache', 'compiler'] + self.compiler_args @@ -70,6 +68,14 @@ def run(self) -> None: if self.compiler_args[i] == '-o': output_files.append(self.compiler_args[i + 1]) + is_linking = [ + is_shared_library_name(output_file_name) for output_file_name in output_files + ] + if is_linking: + cmd_args.extend( + os.environ.get( + COMPILER_WRAPPER_LD_FLAGS_TO_APPEND_ENV_VAR_NAME, '').strip().split()) + if len(output_files) == 1 and output_files[0].endswith('.o'): pp_output_path = None # Perform preprocessing only to ensure we are using the correct include directories. diff --git a/python/yugabyte_db_thirdparty/constants.py b/python/yugabyte_db_thirdparty/constants.py new file mode 100644 index 00000000..6aee2108 --- /dev/null +++ b/python/yugabyte_db_thirdparty/constants.py @@ -0,0 +1,17 @@ +# Copyright (c) Yugabyte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# + + +ADD_CHECKSUM_ARG = '--add-checksum' + +COMPILER_WRAPPER_LD_FLAGS_TO_APPEND_ENV_VAR_NAME = 'YB_COMPILER_WRAPPER_LD_FLAGS_TO_APPEND' diff --git a/python/yugabyte_db_thirdparty/dependency.py b/python/yugabyte_db_thirdparty/dependency.py index 22366e30..cdafa090 100644 --- a/python/yugabyte_db_thirdparty/dependency.py +++ b/python/yugabyte_db_thirdparty/dependency.py @@ -87,6 +87,13 @@ def get_additional_cxx_flags(self, builder: 'BuilderInterface') -> List[str]: def get_additional_ld_flags(self, builder: 'BuilderInterface') -> List[str]: return [] + def get_compiler_wrapper_ld_flags_to_append(self, builder: 'BuilderInterface') -> List[str]: + """ + In some cases, we need to use the compiler_wrapper to add ld flags at the very end of the + compiler wrapper command line. + """ + return [] + def get_additional_assembler_flags(self, builder: 'BuilderInterface') -> List[str]: return [] diff --git a/python/yugabyte_db_thirdparty/download_manager.py b/python/yugabyte_db_thirdparty/download_manager.py index c954125e..1a8a65d7 100644 --- a/python/yugabyte_db_thirdparty/download_manager.py +++ b/python/yugabyte_db_thirdparty/download_manager.py @@ -17,7 +17,7 @@ import subprocess import time -from typing import Optional, List, Dict, cast +from typing import Optional, List, Dict, cast, TYPE_CHECKING from urllib.parse import urlparse from yugabyte_db_thirdparty.archive_handling import ARCHIVE_TYPES @@ -37,6 +37,8 @@ get_temporal_randomized_file_name_suffix, read_file ) +from yugabyte_db_thirdparty.constants import ADD_CHECKSUM_ARG + MAX_FETCH_ATTEMPTS = 20 INITIAL_DOWNLOAD_RETRY_SLEEP_TIME_SEC = 1.0 @@ -194,7 +196,8 @@ def verify_checksum(self, file_name: str, expected_checksum: Optional[str]) -> b if expected_checksum is None: fatal( f"No expected checksum provided for file '{file_basename}'. Consider adding the " - f"following line to thirdparty_src_checksums.txt (or re-run with --add-checksum):\n" + f"following line to thirdparty_src_checksums.txt (or re-run with " + f"{ADD_CHECKSUM_ARG}):\n" f"{real_checksum} {file_basename}\n" ) return real_checksum == expected_checksum diff --git a/python/yugabyte_db_thirdparty/library_checking.py b/python/yugabyte_db_thirdparty/library_checking.py index d5ec8ba4..f856207c 100644 --- a/python/yugabyte_db_thirdparty/library_checking.py +++ b/python/yugabyte_db_thirdparty/library_checking.py @@ -49,10 +49,13 @@ class LibTestBase: # To make sure that we log each allowed pattern no more than once. logged_allowed_patterns: Set[str] + extra_allowed_shared_lib_paths: Set[str] + def __init__(self) -> None: self.tp_installed_dir = os.path.join(YB_THIRDPARTY_DIR, 'installed') self.lib_re_list = [] self.logged_allowed_patterns = set() + self.extra_allowed_shared_lib_paths = set() def init_regex(self) -> None: self.allowed_patterns = compile_re_list(self.lib_re_list) @@ -83,15 +86,20 @@ def check_lib_deps( return status - # overridden in platform specific classes def check_libs_for_file(self, file_path: str) -> bool: + """ + Checks if the given file's shared libraries resolve in a correct way. Overridden in + OS-specific classes. + """ raise NotImplementedError() def run(self) -> None: self.init_regex() heading("Scanning installed executables and libraries...") + for allowed_shared_lib_path in sorted(self.extra_allowed_shared_lib_paths): + log("Extra allowed shared lib path: %s", allowed_shared_lib_path) test_pass = True - # files to examine are much reduced if we look only at bin and lib directories + # Files to examine are much reduced if we look only at bin and lib directories. dir_pattern = re.compile('^(lib|libcxx|[s]bin)$') dirs = [os.path.join(self.tp_installed_dir, type) for type in BUILD_TYPES] for installed_dir in dirs: @@ -115,7 +123,7 @@ def run(self) -> None: log("No problems found with library dependencies.") def add_allowed_shared_lib_paths(self, shared_lib_paths: Set[str]) -> None: - pass + self.extra_allowed_shared_lib_paths |= shared_lib_paths class LibTestMac(LibTestBase): @@ -134,10 +142,6 @@ def __init__(self) -> None: "^\t/usr/lib/", ] - def add_allowed_shared_lib_paths(self, shared_lib_paths: Set[str]) -> None: - # TODO: implement this on macOS for more precise checking of allowed dylib paths. - pass - def check_libs_for_file(self, file_path: str) -> bool: libout = subprocess.check_output(['otool', '-L', file_path]).decode('utf-8') if 'is not an object file' in libout: @@ -190,6 +194,7 @@ def __init__(self) -> None: ] def add_allowed_shared_lib_paths(self, shared_lib_paths: Set[str]) -> None: + super().add_allowed_shared_lib_paths(shared_lib_paths) for shared_lib_path in sorted(shared_lib_paths): self.lib_re_list.append(f".* => {re.escape(shared_lib_path)}/") diff --git a/python/yugabyte_db_thirdparty/toolchain.py b/python/yugabyte_db_thirdparty/toolchain.py index d3679906..d8fe218b 100644 --- a/python/yugabyte_db_thirdparty/toolchain.py +++ b/python/yugabyte_db_thirdparty/toolchain.py @@ -49,14 +49,17 @@ def get_llvm_url(tag: str) -> str: 'llvm12': { 'centos7-x86_64': get_llvm_url('v12.0.1-yb-1-1633099823-bdb147e6-centos7-x86_64'), 'almalinux8-x86_64': get_llvm_url('v12.0.1-yb-1-1633143152-bdb147e6-almalinux8-x86_64'), + }, + 'llvm13': { + 'centos7-x86_64': get_llvm_url('v13.0.1-yb-1-1644383736-191e3a05-centos7-x86_64'), + 'almalinux8-x86_64': get_llvm_url('v13.0.1-yb-1-1644390288-191e3a05-almalinux8-x86_64'), + 'centos8-aarch64': get_llvm_url('v13.0.0-yb-1-1639976983-4b60e646-centos8-aarch64'), } } TOOLCHAIN_TYPES = sorted(TOOLCHAIN_TO_OS_AND_ARCH_TO_URL.keys()) + [ - 'linuxbrew', - 'llvm11_linuxbrew', - 'llvm12_linuxbrew', -] + 'linuxbrew' +] + ['llvm%d_linuxbrew' % v for v in [11, 12, 13]] class Toolchain: diff --git a/python/yugabyte_db_thirdparty/util.py b/python/yugabyte_db_thirdparty/util.py index d6ee06e5..d8556d14 100644 --- a/python/yugabyte_db_thirdparty/util.py +++ b/python/yugabyte_db_thirdparty/util.py @@ -26,6 +26,9 @@ from typing import List, Optional, Any, Dict, Set +SHARED_LIBRARY_EXTENSIONS = ['so', 'dylib'] + + def _detect_yb_thirdparty_dir() -> str: yb_thirdparty_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -305,3 +308,23 @@ def extract_major_version(version_str: str) -> int: 2 ''' return int(version_str.split('.')[0]) + + +def is_shared_library_name(name: str) -> bool: + ''' + >>> is_shared_library_name('libfoo.so') + True + >>> is_shared_library_name('libfoo.dylib') + True + >>> is_shared_library_name('libfoo.so.1') + True + >>> is_shared_library_name('libfoo.dylib.1') + True + >>> is_shared_library_name('soawesome.o') + False + >>> is_shared_library_name('dylibawesome.o') + False + ''' + return any([ + name.endswith('.' + ext) or '.%s.' % ext in name for ext in SHARED_LIBRARY_EXTENSIONS + ]) diff --git a/thirdparty_src_checksums.txt b/thirdparty_src_checksums.txt index cfccdc1a..31b4725e 100644 --- a/thirdparty_src_checksums.txt +++ b/thirdparty_src_checksums.txt @@ -61,5 +61,8 @@ f94c0f816510a95d7521c725e9ddf48bfd600a0f6623d33c9a6a92ec824d8c12 snappy-1.1.3.t 48cc3c73697ff7f6119d0a960fdc3218780cbfb895c187f7f45f2393b16f601f squeasel-8ac777a122fccf0358cb8562e900f8e9edd9ed11-yb-1.tar.gz 2dd06f19e0208c423f7514760150d2f41a8dafdd3aed5384ef2afbfc3df7d900 yb-llvm-v11.1.0-yb-1-1633143292-130bd22e-almalinux8-x86_64.tar.gz 528e37dc711b994aa85765ff2179c9a0ba55dc62b541695ad7991b09dfbc21f1 yb-llvm-v12.0.1-yb-1-1633143152-bdb147e6-almalinux8-x86_64.tar.gz +6ffa3116b89e82cd5039595c9526f79c0dfa10614e3b339bd7250e94f24b6d38 yb-llvm-v13.0.0-yb-1-1639976983-4b60e646-centos8-aarch64.tar.gz c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 zlib-1.2.11.tar.gz 7df6c8f1f462b19a00bf2d4dc191f862cd9a2ea0c10e6ff04b512c07585aa313 crypt_blowfish-1.3.2.tar.gz +a1131358f1f9f819df73fa6bff505f2c49d176e9eef0a3aedd1fdbce3b4630e8 llvm-13.0.0.tar.gz +3437a54d63274790a067eef3b9725238843b37d7a220e16ceb8567d200c81590 llvm-13.0.1-yb-1.tar.gz