From 96592e21e2b186bfba2044dac84a15d0c31ee662 Mon Sep 17 00:00:00 2001 From: apaz Date: Tue, 8 Nov 2022 16:39:29 -0600 Subject: [PATCH] Probe and dlopen() the correct libstdc++ (#46976) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Probe if system libstdc++ is newer than ours If the system libstdc++ is detected to be newer, load it. Otherwise, load the one that we ship. This improves compatibility with external shared libraries that the user might have on their system. Fixes #34276 Co-authored-by: Jameson Nash Co-authored-by: Elliot Saba * Addressed review comments. * Change error handling in wrapper functions Co-authored-by: Jameson Nash * Call write_wrapper three times instead of snprintf Co-authored-by: Jameson Nash * Apply suggestions from code review Co-authored-by: Jameson Nash * Update cli/loader_lib.c Co-authored-by: Jameson Nash * Reordered reading and waiting to avoid a deadlock. * Fixed obvious issues. * Only load libstdc++ preemptively on linux. * Update cli/loader_lib.c Co-authored-by: Jameson Nash * Update cli/loader_lib.c Co-authored-by: Jameson Nash * Specified path to bundled libstdc++ on the command line. * Removed whitespace. * Update cli/Makefile Co-authored-by: Jameson Nash * Handled make install stringreplace. * Correctly quoted stringreplace. * Added -Wl,--enable-new-dtags to prevent DT_RPATH for transitive dependencies * Updated news entry. * Added comment about environment variable. * patched rpath for libgfortran and libLLVM. * Added explaination to Make.inc * Removed trailing space * Removed patchelf for libgfortran, now that BB has been fixed. * Fixed typos and comments Co-authored-by: Max Horn Co-authored-by: Mosè Giordano Co-authored-by: Jameson Nash Co-authored-by: Elliot Saba Co-authored-by: Max Horn (cherry picked from commit eb708d62f897dfbf0a757aa24c30754a492c6514) --- Make.inc | 33 ++++++-- Makefile | 12 ++- NEWS.md | 3 +- cli/Makefile | 6 +- cli/loader_lib.c | 214 +++++++++++++++++++++++++++++++++++++++++++++-- deps/csl.mk | 7 +- 6 files changed, 254 insertions(+), 21 deletions(-) diff --git a/Make.inc b/Make.inc index 41602f354c416..02b73aeadfaf5 100644 --- a/Make.inc +++ b/Make.inc @@ -1147,6 +1147,29 @@ BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(s LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN)))) +# CSL_NEXT_GLIBCXX_VERSION is a triple of the symbols representing support for whatever +# the next libstdc++ version would be. This is used for two things. +# 1. Whether the system libraries are new enough, if we need to use the libs bundled with CSL +# 2. To know which libstdc++ to load at runtime +# We want whichever libstdc++ library is newer, because if we don't it can cause problems. +# While what CSL bundles is quite bleeding-edge compared to what most distros ship, if someone +# tries to build an older branch of Julia, the version of CSL that ships with it may be +# relatively old. This is not a problem for code that is built in BB, but when we build Julia +# with the system compiler, that compiler uses the version of `libstdc++` that it is bundled +# with, and we can get linker errors when trying to run that `julia` executable with the +# `libstdc++` that comes from the (now old) BB-built CSL. +# To fix this, we take note when the system `libstdc++.so` is newer than whatever we +# would get from CSL (by searching for a `GLIBCXX_X.Y.Z` symbol that does not exist +# in our CSL, but would in a newer one), and default to `USE_BINARYBUILDER_CSL=0` in +# this case. This ensures that we link against a version with the symbols required. +# We also check the system libstdc++ at runtime in the cli loader library, and +# load it if it contains the version symbol that indicates that it is newer than the one +# shipped with CSL. Although we do not depend on any of the symbols, it is entirely +# possible that a user might choose to install a library which depends on symbols provided +# by a newer libstdc++. Without runtime detection, those libraries would break. +CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.31|GLIBCXX_3\.5\.|GLIBCXX_4\. + + # This is the set of projects that BinaryBuilder dependencies are hooked up for. # Note: we explicitly _do not_ define `CSL` here, since it requires some more # advanced techniques to decide whether it should be installed from a BB source @@ -1203,18 +1226,16 @@ ifneq (,$(filter $(OS),WINNT emscripten)) RPATH := RPATH_ORIGIN := RPATH_ESCAPED_ORIGIN := - RPATH_LIB := else ifeq ($(OS), Darwin) RPATH := -Wl,-rpath,'@executable_path/$(build_libdir_rel)' RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' RPATH_ESCAPED_ORIGIN := $(RPATH_ORIGIN) - RPATH_LIB := -Wl,-rpath,'@loader_path/' else - RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin - RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin - RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin -Wl,-rpath-link,$(build_shlibdir) - RPATH_LIB := -Wl,-rpath,'$$ORIGIN/' -Wl,-z,origin + RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin -Wl,--enable-new-dtags + RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin -Wl,--enable-new-dtags + RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin -Wl,-rpath-link,$(build_shlibdir) -Wl,--enable-new-dtags endif +RPATH_LIB := $(RPATH_ORIGIN) # --whole-archive ifeq ($(OS), Darwin) diff --git a/Makefile b/Makefile index 57b5953107914..94df626014059 100644 --- a/Makefile +++ b/Makefile @@ -225,7 +225,7 @@ endif # Note that we disable MSYS2's path munging here, as otherwise # it replaces our `:`-separated list as a `;`-separated one. define stringreplace - MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)" + MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep "$2" | awk '{print $$1;}') "$3" 255 "$(call cygpath_w,$1)" endef @@ -374,6 +374,16 @@ ifeq ($(BUNDLE_DEBUG_LIBS),1) endif endif + # Fix rpaths for dependencies. This should be fixed in BinaryBuilder later. +ifeq ($(OS), Linux) + -$(PATCHELF) --set-rpath '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT) +endif + + # Replace libstdc++ path, which is also moving from `lib` to `../lib/julia`. +ifeq ($(OS),Linux) + $(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT),\*libstdc++\.so\.6$$,*$(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libstdc++.so.6)) +endif + ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS)) # Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS) diff --git a/NEWS.md b/NEWS.md index 62683964eb884..da7fcb356b5a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -244,7 +244,8 @@ Deprecated or removed External dependencies --------------------- - +* On Linux, now autodetects the system libstdc++ version, and automatically loads the system library if it is newer. The old behavior of loading the bundled libstdc++ regardless of the system version obtained by setting the environment variable `JULIA_PROBE_LIBSTDCXX=0`. +* Removed `RPATH` from the julia binary. On Linux this may break libraries that have failed to set `RUNPATH`. Tooling Improvements --------------------- diff --git a/cli/Makefile b/cli/Makefile index 16b479954c724..bfbd2d8c75262 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -13,6 +13,8 @@ LOADER_LDFLAGS = $(JLDFLAGS) -ffreestanding -L$(build_shlibdir) -L$(build_libdir ifeq ($(OS),WINNT) LOADER_CFLAGS += -municode -mconsole -nostdlib -fno-stack-check -fno-stack-protector -mno-stack-arg-probe +else ifeq ($(OS),Linux) +LOADER_CFLAGS += -DGLIBCXX_LEAST_VERSION_SYMBOL=\"$(shell echo "$(CSL_NEXT_GLIBCXX_VERSION)" | cut -d'|' -f1 | sed 's/\\//g')\" endif ifeq ($(OS),WINNT) @@ -111,7 +113,7 @@ endif $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_OBJS) $(SRCDIR)/list_strip_symbols.h | $(build_shlibdir) $(build_libdir) @$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -DLIBRARY_EXPORTS -shared $(SHIPFLAGS) $(LIB_OBJS) -o $@ \ - $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(RPATH_LIB) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT))) + $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT))) @$(INSTALL_NAME_CMD)libjulia.$(SHLIB_EXT) $@ ifeq ($(OS), WINNT) @# Note that if the objcopy command starts getting too long, we can use `@file` to read @@ -121,7 +123,7 @@ endif $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_DOBJS) $(SRCDIR)/list_strip_symbols.h | $(build_shlibdir) $(build_libdir) @$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -DLIBRARY_EXPORTS -shared $(DEBUGFLAGS) $(LIB_DOBJS) -o $@ \ - $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(RPATH_LIB) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT))) + $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT))) @$(INSTALL_NAME_CMD)libjulia-debug.$(SHLIB_EXT) $@ ifeq ($(OS), WINNT) @$(call PRINT_ANALYZE, $(OBJCOPY) $(build_libdir)/$(notdir $@).tmp.a $(STRIP_EXPORTED_FUNCS) $(build_libdir)/$(notdir $@).a && rm $(build_libdir)/$(notdir $@).tmp.a) diff --git a/cli/loader_lib.c b/cli/loader_lib.c index 0301b6eedde62..0069f9e793a1e 100644 --- a/cli/loader_lib.c +++ b/cli/loader_lib.c @@ -16,7 +16,7 @@ extern "C" { #endif // Save DEP_LIBS to a variable that is explicitly sized for expansion -static char dep_libs[1024] = DEP_LIBS; +static char dep_libs[1024] = "\0" DEP_LIBS; JL_DLLEXPORT void jl_loader_print_stderr(const char * msg) { @@ -33,7 +33,6 @@ void jl_loader_print_stderr3(const char * msg1, const char * msg2, const char * /* Wrapper around dlopen(), with extra relative pathing thrown in*/ static void * load_library(const char * rel_path, const char * src_dir, int err) { void * handle = NULL; - // See if a handle is already open to the basename const char *basename = rel_path + strlen(rel_path); while (basename-- > rel_path) @@ -147,6 +146,174 @@ JL_DLLEXPORT const char * jl_get_libdir() return lib_dir; } +// On Linux, it can happen that the system has a newer libstdc++ than the one we ship, +// which can break loading of some system libraries: . +// As a fix, on linux we probe the system libstdc++ to see if it is newer, and then load it if it is. +// Otherwise, we load the bundled one. This improves compatibility with third party dynamic libs that +// may depend on symbols exported by the system libstdxc++. +#ifdef _OS_LINUX_ +#ifndef GLIBCXX_LEAST_VERSION_SYMBOL +#warning GLIBCXX_LEAST_VERSION_SYMBOL should always be defined in the makefile. +#define GLIBCXX_LEAST_VERSION_SYMBOL "GLIBCXX_a.b.c" /* Appease the linter */ +#endif + +#include +#include + +// write(), but handle errors and avoid EINTR +static void write_wrapper(int fd, const char *str, size_t len) +{ + size_t written_sofar = 0; + while (len) { + ssize_t bytes_written = write(fd, str + written_sofar, len); + if (bytes_written == -1 && errno == EINTR) continue; + if (bytes_written == -1 && errno != EINTR) { + perror("(julia) child libstdcxxprobe write"); + _exit(1); + } + len -= bytes_written; + written_sofar += bytes_written; + } +} + +// read(), but handle errors and avoid EINTR +static void read_wrapper(int fd, char **ret, size_t *ret_len) +{ + // Allocate an initial buffer + size_t len = JL_PATH_MAX; + char *buf = (char *)malloc(len + 1); + if (!buf) { + perror("(julia) malloc"); + exit(1); + } + + // Read into it, reallocating as necessary + size_t have_read = 0; + while (1) { + ssize_t n = read(fd, buf + have_read, len - have_read); + have_read += n; + if (n == 0) break; + if (n == -1 && errno != EINTR) { + perror("(julia) libstdcxxprobe read"); + exit(1); + } + if (n == -1 && errno == EINTR) continue; + if (have_read == len) { + buf = (char *)realloc(buf, 1 + (len *= 2)); + if (!buf) { + perror("(julia) realloc"); + exit(1); + } + } + } + + *ret = buf; + *ret_len = have_read; +} + +// Return the path to the libstdcxx to load. +// If the path is found, return it. +// Otherwise, print the error and exit. +// The path returned must be freed. +static char *libstdcxxprobe(void) +{ + // Create the pipe and child process. + int fork_pipe[2]; + int ret = pipe(fork_pipe); + if (ret == -1) { + perror("(julia) Error during libstdcxxprobe: pipe"); + exit(1); + } + pid_t pid = fork(); + if (pid == -1) { + perror("Error during libstdcxxprobe:\nfork"); + exit(1); + } + if (pid == (pid_t) 0) { // Child process. + close(fork_pipe[0]); + + // Open the first available libstdc++.so. + // If it can't be found, report so by exiting zero. + // The star is there to prevent the compiler from merging constants + // with "\0*libstdc++.so.6", which we string replace inside the .so during + // make install. + void *handle = dlopen("libstdc++.so.6\0*", RTLD_LAZY); + if (!handle) { + _exit(0); + } + + // See if the version is compatible + char *dlerr = dlerror(); // clear out dlerror + void *sym = dlsym(handle, GLIBCXX_LEAST_VERSION_SYMBOL); + dlerr = dlerror(); + if (dlerr) { + // We can't use the library that was found, so don't write anything. + // The main process will see that nothing was written, + // then exit the function and return null. + _exit(0); + } + + // No error means the symbol was found, we can use this library. + // Get the path to it, and write it to the parent process. + struct link_map *lm; + ret = dlinfo(handle, RTLD_DI_LINKMAP, &lm); + if (ret == -1) { + char *errbuf = dlerror(); + char *errdesc = (char*)"Error during libstdcxxprobe in child process:\ndlinfo: "; + write_wrapper(STDERR_FILENO, errdesc, strlen(errdesc)); + write_wrapper(STDERR_FILENO, errbuf, strlen(errbuf)); + write_wrapper(STDERR_FILENO, "\n", 1); + _exit(1); + } + char *libpath = lm->l_name; + write_wrapper(fork_pipe[1], libpath, strlen(libpath)); + _exit(0); + } + else { // Parent process. + close(fork_pipe[1]); + + // Read the absolute path to the lib from the child process. + char *path; + size_t pathlen; + read_wrapper(fork_pipe[0], &path, &pathlen); + + // Close the read end of the pipe + close(fork_pipe[0]); + + // Wait for the child to complete. + while (1) { + int wstatus; + pid_t npid = waitpid(pid, &wstatus, 0); + if (npid == -1) { + if (errno == EINTR) continue; + if (errno != EINTR) { + perror("Error during libstdcxxprobe in parent process:\nwaitpid"); + exit(1); + } + } + else if (!WIFEXITED(wstatus)) { + const char *err_str = "Error during libstdcxxprobe in parent process:\n" + "The child process did not exit normally.\n"; + size_t err_strlen = strlen(err_str); + write_wrapper(STDERR_FILENO, err_str, err_strlen); + exit(1); + } + else if (WEXITSTATUS(wstatus)) { + // The child has printed an error and exited, so the parent should exit too. + exit(1); + } + break; + } + + if (!pathlen) { + free(path); + return NULL; + } + return path; + } +} +#endif + void * libjulia_internal = NULL; __attribute__((constructor)) void jl_load_libjulia_internal(void) { // Only initialize this once @@ -155,11 +322,43 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { } // Introspect to find our own path - const char * lib_dir = jl_get_libdir(); + const char *lib_dir = jl_get_libdir(); // Pre-load libraries that libjulia-internal needs. - int deps_len = strlen(dep_libs); - char * curr_dep = &dep_libs[0]; + int deps_len = strlen(&dep_libs[1]); + char *curr_dep = &dep_libs[1]; + + void *cxx_handle; + +#if defined(_OS_LINUX_) + int do_probe = 1; + int done_probe = 0; + char *probevar = getenv("JULIA_PROBE_LIBSTDCXX"); + if (probevar) { + if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0) + do_probe = 1; + else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0) + do_probe = 0; + } + if (do_probe) { + char *cxxpath = libstdcxxprobe(); + if (cxxpath) { + cxx_handle = dlopen(cxxpath, RTLD_LAZY); + char *dlr = dlerror(); + if (dlr) { + jl_loader_print_stderr("ERROR: Unable to dlopen(cxxpath) in parent!\n"); + jl_loader_print_stderr3("Message: ", dlr, "\n"); + exit(1); + } + free(cxxpath); + done_probe = 1; + } + } + if (!done_probe) { + const static char bundled_path[256] = "\0*libstdc++.so.6"; + load_library(&bundled_path[2], lib_dir, 1); + } +#endif // We keep track of "special" libraries names (ones whose name is prefixed with `@`) // which are libraries that we want to load in some special, custom way, such as @@ -183,7 +382,8 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { } special_library_names[special_idx] = curr_dep + 1; special_idx += 1; - } else { + } + else { load_library(curr_dep, lib_dir, 1); } @@ -272,7 +472,7 @@ JL_DLLEXPORT int jl_load_repl(int argc, char * argv[]) { } #ifdef _OS_WINDOWS_ -int __stdcall DllMainCRTStartup(void* instance, unsigned reason, void* reserved) { +int __stdcall DllMainCRTStartup(void *instance, unsigned reason, void *reserved) { setup_stdio(); // Because we override DllMainCRTStartup, we have to manually call our constructor methods diff --git a/deps/csl.mk b/deps/csl.mk index 0df2b92b6ce39..dd782880d21fd 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -12,17 +12,16 @@ endef # CSL bundles lots of system compiler libraries, and while it is quite bleeding-edge # as compared to what most distros ship, if someone tries to build an older branch, -# the version of CSL that ships with that branch may become relatively old. This is -# not a problem for code that is built in BB, but when we build Julia with the system +# the version of CSL that ships with that branch may be relatively old. This is not +# a problem for code that is built in BB, but when we build Julia with the system # compiler, that compiler uses the version of `libstdc++` that it is bundled with, -# and we can get linker errors when trying to run that `julia` executable with the +# and we can get linker errors when trying to run that `julia` executable with the # `libstdc++` that comes from the (now old) BB-built CSL. # # To fix this, we take note when the system `libstdc++.so` is newer than whatever we # would get from CSL (by searching for a `GLIBCXX_3.4.X` symbol that does not exist # in our CSL, but would in a newer one), and default to `USE_BINARYBUILDER_CSL=0` in # this case. -CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.31|GLIBCXX_3\.5\.|GLIBCXX_4\. # First, check to see if BB is disabled on a global setting ifeq ($(USE_BINARYBUILDER),0)