Adding krb5 (#104)

Based on @nathanhjli's work in #100

Other changes:
- Removing the CentOS 7 Clang 7 build and compiler flags for Clang 7.
- Simplifying instantiation of Dependency objects by module name in builder.py for modules in build_definitions that only have one Dependency subclass.
- Enable compiler wrapper as needed for each dependency.

.github/workflows/ci.yml

@@ -41,13 +41,6 @@ jobs:
               --devtoolset=9
               --expected-major-compiler-version=9
 
-          - name: centos7-x86_64-clang7
-            os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7:v2021-08-27T03_10_19
-            build_thirdparty_args: >-
-              --toolchain=llvm7
-              --expected-major-compiler-version=7
-
           - name: centos7-x86_64-clang11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
             docker_image: yugabyteci/yb_build_infra_centos7:v2021-08-27T03_10_19

patches/krb5-1.19.3-use-ldflags-for-test.patch

@@ -0,0 +1,13 @@
+diff --git src/tests/asn.1/Makefile.in src/tests/asn.1/Makefile.in
+index eabe0bd..b0a3b74 100644
+--- src/tests/asn.1/Makefile.in
++++ src/tests/asn.1/Makefile.in
+@@ -29,7 +29,7 @@ krb5_decode_leak: $(LEAKOBJS) $(KRB5_BASE_DEPLIBS)
+ 	$(CC_LINK) -o krb5_decode_leak $(LEAKOBJS) $(KRB5_BASE_LIBS)
+
+ t_trval: t_trval.o
+-	$(CC) -o t_trval $(ALL_CFLAGS) t_trval.o
++	$(CC) -o t_trval $(ALL_CFLAGS) $(LDFLAGS) t_trval.o
+
+ check: check-encode check-encode-trval check-decode check-leak
+

python/build_definitions/__init__.py

@@ -18,10 +18,13 @@
 import importlib
 import pkgutil
 
-from typing import Any, List, Dict, Union
+from typing import Any, List, Dict, Union, TYPE_CHECKING
+
 from yugabyte_db_thirdparty.custom_logging import log
 from yugabyte_db_thirdparty.archive_handling import make_archive_name
 
+if TYPE_CHECKING:
+    from yugabyte_db_thirdparty.dependency import Dependency
 
 # -------------------------------------------------------------------------------------------------
 # Build groups
@@ -105,5 +108,34 @@ def get_build_def_module(submodule_name: str) -> Any:
     return getattr(sys.modules['build_definitions'], submodule_name)
 
 
+def get_dependency_by_submodule_name(module_name: str) -> 'Dependency':
+    build_def_module = get_build_def_module(module_name)
+    candidate_classes: List[Any] = []
+    for field_name in dir(build_def_module):
+        field_value = getattr(build_def_module, field_name)
+        if isinstance(field_value, type):
+            try:
+                class_name = getattr(field_value, '__name__')
+            except AttributeError:
+                continue
+            if class_name != 'Dependency' and class_name.endswith('Dependency'):
+                candidate_classes.append(field_value)
+
+    if not candidate_classes:
+        raise ValueError(
+            "Could not find a ...Dependency class in module %s that starts with submodule name" %
+            module_name)
+
+    if len(candidate_classes) > 1:
+        raise ValueError("Found too many classes with names ending with Dependency in module "
+                         "%s: %s", module_name, sorted(
+                             [cl.__name__ for cl in candidate_classes]))
+    return candidate_classes[0]()
+
+
+def get_deps_from_module_names(module_names: List[str]) -> List['Dependency']:
+    return [get_dependency_by_submodule_name(module_name) for module_name in module_names]
+
+
 def validate_build_type(build_type: str) -> None:
     assert build_type in BUILD_TYPES, f"Invalid build type: {build_type}"

python/build_definitions/crcutil.py

@@ -30,14 +30,14 @@ def __init__(self) -> None:
         self.patches = ['crcutil-fix-offsetof.patch']
 
     def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
-        if (builder.compiler_choice.building_with_clang(builder.build_type) or
-                platform.uname().processor == 'aarch64'):
-            return []
-        # -mcrc32 (https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html)
-        # This option enables built-in functions __builtin_ia32_crc32qi, __builtin_ia32_crc32hi,
-        # __builtin_ia32_crc32si and __builtin_ia32_crc32di to generate the crc32 machine
-        # instruction.
-        return ['-mcrc32']
+        if (builder.compiler_choice.compiler_type == 'gcc' and
+                platform.uname().processor == 'x86_64'):
+            # -mcrc32 (https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html)
+            # This option enables built-in functions __builtin_ia32_crc32qi, __builtin_ia32_crc32hi,
+            # __builtin_ia32_crc32si and __builtin_ia32_crc32di to generate the crc32 machine
+            # instruction.
+            return ['-mcrc32']
+        return []
 
     def build(self, builder: BuilderInterface) -> None:
         log_prefix = builder.log_prefix(self)

python/build_definitions/krb5.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) YugaByte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import sys
+
+from yugabyte_db_thirdparty.build_definition_helpers import *  # noqa
+
+
+class Krb5Dependency(Dependency):
+    def __init__(self) -> None:
+        super(Krb5Dependency, self).__init__(
+            'krb5',
+            '1.19.3',
+            'https://kerberos.org/dist/krb5/1.19/krb5-{0}.tar.gz',
+            BUILD_GROUP_INSTRUMENTED)
+        self.copy_sources = True
+        self.patches = ['krb5-1.19.3-use-ldflags-for-test.patch']
+        self.patch_strip = 0
+
+    def get_additional_ld_flags(self, builder: BuilderInterface) -> List[str]:
+        flags: List[str] = super().get_additional_ld_flags(builder)
+        if builder.compiler_choice.is_linux_clang1x():
+            if builder.build_type == BUILD_TYPE_ASAN:
+                # Needed to find dlsym.
+                flags.append('-ldl')
+        return flags
+
+    def get_compiler_wrapper_ld_flags_to_remove(self, builder: BuilderInterface) -> Set[str]:
+        return {'-Wl,--no-undefined'}
+
+    def build(self, builder: BuilderInterface) -> None:
+        extra_args = [
+            '--disable-nls',  # Remove the dependency on gettext.
+        ]
+        if builder.build_type in [BUILD_TYPE_ASAN]:
+            extra_args.append('--enable-asan')
+        builder.build_with_configure(
+            log_prefix=builder.log_prefix(self),
+            src_subdir_name='src',
+            extra_args=extra_args,
+        )

python/yugabyte_db_thirdparty/build_definition_helpers.py

@@ -34,5 +34,5 @@
     BUILD_TYPE_TSAN,
 )
 
-from typing import List, Dict, Any
+from typing import List, Dict, Set, Any
 from sys_detection import is_macos