Skip to content

Commit

Permalink
Add 'fake' JLLs for all binary dependencies
Browse files Browse the repository at this point in the history 
JLLs provide the binary interaction interface that both packages and the
Pkg resolver require in order to properly version and interact with the
binary dependencies shipped with Julia.  This PR changes all binary
dependencies to carry along a JLL package that dlopen()'s the binary
that ships along with Julia.  These JLL packages are not the true JLL
packages that are generated by BinaryBuilder, but are instead "fake"
JLLs that simply `dlopen(filename)` and rely upon the default dlopen
path to find the appropriate libraries, which affords the opportunity
for system libraries and bundled-with-julia libraries alike to both be
represented through the JLL interface.
  • Loading branch information
@staticfloat
staticfloat committed Nov 19, 2020
1 parent 77a1fe8 commit 5f43f3e
Show file tree
Hide file tree
Showing 93 changed files with 1,662 additions and 387 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@
/perf*
.DS_Store
.idea/*
.vscode/*
40 changes: 26 additions & 14 deletions Make.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ OPENBLAS_USE_THREAD:=1
# Flags for using libraries available on the system instead of building them.
# Please read the notes around usage of SYSTEM flags in README.md
# Issues resulting from use of SYSTEM versions will generally not be accepted.
USE_SYSTEM_CSL:=0
USE_SYSTEM_LLVM:=0
USE_SYSTEM_LIBUNWIND:=0
DISABLE_LIBUNWIND:=0
Expand Down Expand Up @@ -93,6 +94,15 @@ WITH_GC_DEBUG_ENV := 0
# Prevent picking up $ARCH from the environment variables
ARCH:=


# Literal values that are hard to use in Makefiles otherwise:
define newline # a literal \n


endef
COMMA:=,
SPACE:=$(eval) $(eval)

# We need python for things like BB triplet recognition and relative path computation.
# We don't really care about version, generally, so just find something that works:
PYTHON := "$(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo "{python|python3|python2} not found")"
Expand Down Expand Up @@ -999,19 +1009,25 @@ ifeq ($(USE_SYSTEM_LLVM), 1)
JCPPFLAGS+=-DSYSTEM_LLVM
endif # SYSTEM_LLVM

# Windows builds need a little help finding the LLVM libraries for llvm-config
LLVM_CONFIG_PATH_FIX :=
ifeq ($(OS),WINNT)
LLVM_CONFIG_PATH_FIX := PATH="$(PATH):$(build_bindir)"
endif

ifeq ($(BUILD_OS),$(OS))
LLVM_CONFIG_HOST := $(LLVM_CONFIG)
else
LLVM_CONFIG_HOST := $(basename $(LLVM_CONFIG))-host$(BUILD_EXE)
ifeq (exists, $(shell [ -f '$(LLVM_CONFIG_HOST)' ] && echo exists ))
ifeq ($(shell $(LLVM_CONFIG_HOST) --version),3.3)
ifeq ($(shell $(LLVM_CONFIG_PATH_FIX) $(LLVM_CONFIG_HOST) --version),3.3)
# llvm-config-host <= 3.3 is broken, use llvm-config instead (in an emulator)
# use delayed expansion (= not :=) because spawn isn't defined until later
LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG))
LLVM_CONFIG_HOST = $(LLVM_CONFIG_PATH_FIX) $(call spawn,$(LLVM_CONFIG))
endif
else
# llvm-config-host does not exist (cmake build)
LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG))
LLVM_CONFIG_HOST = $(LLVM_CONFIG_PATH_FIX) $(call spawn,$(LLVM_CONFIG))
endif
endif

Expand Down Expand Up @@ -1129,8 +1145,14 @@ endif
USE_BINARYBUILDER ?= 0
endif

# Auto-detect triplet once, create different versions that we use as defaults below for each BB install target
BB_TRIPLET_LIBGFORTRAN_CXXABI := $(shell $(call invoke_python,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)) "$(shell $(FC) --version | head -1)" "$(or $(shell echo '\#include <string>' | $(CXX) $(CXXFLAGS) -x c++ -dM -E - | grep _GLIBCXX_USE_CXX11_ABI | awk '{ print $$3 }' ),1)")
BB_TRIPLET_LIBGFORTRAN := $(subst $(SPACE),-,$(filter-out cxx%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI))))
BB_TRIPLET_CXXABI := $(subst $(SPACE),-,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI))))
BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI)))))

# This is the set of projects that BinaryBuilder dependencies are hooked up for.
BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP CSL
define SET_BB_DEFAULT
# First, check to see if BB is disabled on a global setting
ifeq ($$(USE_BINARYBUILDER),0)
Expand Down Expand Up @@ -1576,18 +1598,8 @@ PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1)
PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1)
endif
define newline # a literal \n
endef
# Makefile debugging trick:
# call print-VARIABLE to see the runtime value of any variable
# (hardened against any special characters appearing in the output)
print-%:
@echo '$*=$(subst ','\'',$(subst $(newline),\n,$($*)))'

# Literal values that are hard to use in Makefiles otherwise:
COMMA:=,
SPACE:=$(eval) $(eval)
88 changes: 14 additions & 74 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ endif
julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps

julia-stdlib: | $(DIRS)
# `julia-stdlib` depends on `julia-deps` so that the fake JLL stdlibs can copy in their Artifacts.toml files.
julia-stdlib: | $(DIRS) julia-deps
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib

julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl
Expand Down Expand Up @@ -163,17 +164,18 @@ endif
JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest
ifeq ($(USE_GPL_LIBS), 1)
JL_PRIVATE_LIBS-0 += libsuitesparse_wrapper
JL_PRIVATE_LIBS-$(USE_SYSTEM_SUITESPARSE) += libamd libcamd libccolamd libcholmod libcolamd libumfpack libspqr libsuitesparseconfig
JL_PRIVATE_LIBS-$(USE_SYSTEM_SUITESPARSE) += libamd libbtf libcamd libccolamd libcholmod libcolamd libklu libldl librbio libspqr libsuitesparseconfig libumfpack
endif
JL_PRIVATE_LIBS-$(USE_SYSTEM_PCRE) += libpcre2-8
JL_PRIVATE_LIBS-$(USE_SYSTEM_DSFMT) += libdSFMT
JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp
JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp libgmpxx
JL_PRIVATE_LIBS-$(USE_SYSTEM_MPFR) += libmpfr
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSSH2) += libssh2
JL_PRIVATE_LIBS-$(USE_SYSTEM_NGHTTP2) += libnghttp2
JL_PRIVATE_LIBS-$(USE_SYSTEM_MBEDTLS) += libmbedtls libmbedcrypto libmbedx509
JL_PRIVATE_LIBS-$(USE_SYSTEM_CURL) += libcurl
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBGIT2) += libgit2
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBUV) += libuv
ifeq ($(OS),WINNT)
JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib
else
Expand All @@ -197,91 +199,29 @@ ifneq ($(LIBLAPACKNAME),$(LIBBLASNAME))
JL_PRIVATE_LIBS-$(USE_SYSTEM_LAPACK) += $(LIBLAPACKNAME)
endif

JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libgfortran libquadmath libstdc++ libgcc_s libgomp libssp libatomic
ifeq ($(OS),Darwin)
ifeq ($(USE_SYSTEM_BLAS),1)
ifeq ($(USE_SYSTEM_LAPACK),0)
JL_PRIVATE_LIBS-0 += libgfortblas
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libc++
endif
endif
endif

# On FreeBSD, /lib/libgcc_s.so.1 is incompatible with Fortran; to use Fortran on FreeBSD,
# we need to link to the libgcc_s that ships with the same GCC version used by libgfortran.
# To work around this, we copy the GCC libraries we need, namely libgfortran, libgcc_s,
# and libquadmath, into our build library directory, $(build_libdir). We also add them to
# JL_PRIVATE_LIBS-0 so that they know where they need to live at install time.
ifeq ($(OS),FreeBSD)
define std_so
julia-deps: | $$(build_libdir)/$(1).so
$$(build_libdir)/$(1).so: | $$(build_libdir)
$$(INSTALL_M) $$(GCCPATH)/$(1).so* $$(build_libdir)
JL_PRIVATE_LIBS-0 += $(1)
endef

$(eval $(call std_so,libgfortran))
$(eval $(call std_so,libgcc_s))
$(eval $(call std_so,libquadmath))
endif # FreeBSD

ifeq ($(OS),WINNT)
# find the standard .dll folders
ifeq ($(XC_HOST),)
STD_LIB_PATH ?= $(PATH)
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libwinpthread
else
STD_LIB_PATH := $(shell LANG=C $(CC) -print-search-dirs | grep '^programs: =' | sed -e "s/^programs: =//")
STD_LIB_PATH += :$(shell LANG=C $(CC) -print-search-dirs | grep '^libraries: =' | sed -e "s/^libraries: =//")
ifneq (,$(findstring CYGWIN,$(BUILD_OS))) # the cygwin-mingw32 compiler lies about it search directory paths
STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g")
endif
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libpthread
endif

pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2)))))

define std_dll
julia-deps-libs: | $$(build_bindir)/lib$(1).dll $$(build_depsbindir)/lib$(1).dll
$$(build_bindir)/lib$(1).dll: | $$(build_bindir)
cp $$(or $$(call pathsearch,lib$(1).dll,$$(STD_LIB_PATH)),$$(error can't find lib$1.dll)) $$(build_bindir)
$$(build_depsbindir)/lib$(1).dll: | $$(build_depsbindir)
cp $$(or $$(call pathsearch,lib$(1).dll,$$(STD_LIB_PATH)),$$(error can't find lib$1.dll)) $$(build_depsbindir)
JL_TARGETS += $(1)
endef
julia-deps: julia-deps-libs

# Given a list of space-separated libraries, return the first library name that is
# correctly found through `pathsearch`.
define select_std_dll
$(firstword $(foreach name,$(1),$(if $(call pathsearch,lib$(name).dll,$(STD_LIB_PATH)),$(name),)))
endef

$(eval $(call std_dll,$(call select_std_dll,gfortran-3 gfortran-4 gfortran-5)))
$(eval $(call std_dll,quadmath-0))
$(eval $(call std_dll,stdc++-6))
ifeq ($(ARCH),i686)
$(eval $(call std_dll,gcc_s_sjlj-1))
else
$(eval $(call std_dll,gcc_s_seh-1))
ifeq ($(OS),Darwin)
ifeq ($(USE_SYSTEM_BLAS),1)
ifeq ($(USE_SYSTEM_LAPACK),0)
JL_PRIVATE_LIBS-0 += libgfortblas
endif
endif
$(eval $(call std_dll,ssp-0))
$(eval $(call std_dll,winpthread-1))
$(eval $(call std_dll,atomic-1))
endif


define stringreplace
$(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)"
endef

# Run fixup-libgfortran on all platforms but Windows and FreeBSD. On FreeBSD we
# pull in the GCC libraries earlier and use them for the build to make sure we
# don't inadvertently link to /lib/libgcc_s.so.1, which is incompatible with
# libgfortran, and on Windows we copy them in earlier as well.
ifeq (,$(findstring $(OS),FreeBSD WINNT))
julia-base: $(build_libdir)/libgfortran*.$(SHLIB_EXT)*
$(build_libdir)/libgfortran*.$(SHLIB_EXT)*: | $(build_libdir) julia-deps
-$(CUSTOM_LD_LIBRARY_PATH) PATH="$(PATH):$(build_depsbindir)" PATCHELF="$(PATCHELF)" FC="$(FC)" $(JULIAHOME)/contrib/fixup-libgfortran.sh --verbose $(build_libdir)
JL_PRIVATE_LIBS-0 += libgfortran libgcc_s libquadmath
endif


install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
ifeq ($(BUNDLE_DEBUG_LIBS),1)
Expand Down
1 change: 1 addition & 0 deletions base/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ ifeq ($(DARWIN_FRAMEWORK), 1)
else
@echo "const DARWIN_FRAMEWORK = false" >> $@
endif
@echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@

@# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl,
@# ensuring we rebuild the system image as infrequently as possible
Expand Down
60 changes: 34 additions & 26 deletions base/binaryplatforms.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,15 +232,15 @@ function validate_tags(tags::Dict)
throw_version_number("libgfortran_version")
end

# Validate `libstdcxx_version` is a parsable `VersionNumber`
if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing
throw_version_number("libstdcxx_version")
end

# Validate `cxxstring_abi` is one of the two valid options:
if "cxxstring_abi" in keys(tags) && tags["cxxstring_abi"] ∉ ("cxx03", "cxx11")
throw_invalid_key("cxxstring_abi")
end

# Validate `libstdcxx_version` is a parsable `VersionNumber`
if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing
throw_version_number("libstdcxx_version")
end
end

function set_compare_strategy!(p::Platform, key::String, f::Function)
Expand Down Expand Up @@ -508,12 +508,12 @@ function triplet(p::AbstractPlatform)
if libgfortran_version(p) !== nothing
str = string(str, "-libgfortran", libgfortran_version(p).major)
end
if libstdcxx_version(p) !== nothing
str = string(str, "-libstdcxx", libstdcxx_version(p).patch)
end
if cxxstring_abi(p) !== nothing
str = string(str, "-", cxxstring_abi(p))
end
if libstdcxx_version(p) !== nothing
str = string(str, "-libstdcxx", libstdcxx_version(p).patch)
end

# Tack on all extra tags
for (tag, val) in tags(p)
Expand Down Expand Up @@ -637,15 +637,15 @@ const libgfortran_version_mapping = Dict(
"libgfortran4" => "(-libgfortran4)|(-gcc7)",
"libgfortran5" => "(-libgfortran5)|(-gcc8)",
)
const libstdcxx_version_mapping = Dict{String,String}(
"libstdcxx_nothing" => "",
"libstdcxx" => "-libstdcxx\\d+",
)
const cxxstring_abi_mapping = Dict(
"cxxstring_nothing" => "",
"cxx03" => "-cxx03",
"cxx11" => "-cxx11",
)
const libstdcxx_version_mapping = Dict{String,String}(
"libstdcxx_nothing" => "",
"libstdcxx" => "-libstdcxx\\d+",
)

"""
parse(::Type{Platform}, triplet::AbstractString)
Expand All @@ -667,8 +667,8 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
c(call_abi_mapping),
# Next, optional things, like libgfortran/libstdcxx/cxxstring abi
c(libgfortran_version_mapping),
c(libstdcxx_version_mapping),
c(cxxstring_abi_mapping),
c(libstdcxx_version_mapping),
# Finally, the catch-all for extended tags
"(?<tags>(?:-[^-]+\\+[^-]+)*)?",
"\$",
Expand Down Expand Up @@ -736,8 +736,8 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
libc,
call_abi,
libgfortran_version,
libstdcxx_version,
cxxstring_abi,
libstdcxx_version,
os_version,
tags...,
)
Expand Down Expand Up @@ -929,25 +929,33 @@ detect compiler ABI values such as `libgfortran_version`, `libstdcxx_version` an
we have much of that built.
"""
function host_triplet()
str = Sys.MACHINE
libgfortran_version = detect_libgfortran_version()
if libgfortran_version !== nothing
str = string(str, "-libgfortran", libgfortran_version.major)
str = Base.BUILD_TRIPLET

if !occursin("-libgfortran", str)
libgfortran_version = detect_libgfortran_version()
if libgfortran_version !== nothing
str = string(str, "-libgfortran", libgfortran_version.major)
end
end

libstdcxx_version = detect_libstdcxx_version()
if libstdcxx_version !== nothing
str = string(str, "-libstdcxx", libstdcxx_version.patch)
if !occursin("-cxx", str)
cxxstring_abi = detect_cxxstring_abi()
if cxxstring_abi !== nothing
str = string(str, "-", cxxstring_abi)
end
end

cxxstring_abi = detect_cxxstring_abi()
if cxxstring_abi !== nothing
str = string(str, "-", cxxstring_abi)
if !occursin("-libstdcxx", str)
libstdcxx_version = detect_libstdcxx_version()
if libstdcxx_version !== nothing
str = string(str, "-libstdcxx", libstdcxx_version.patch)
end
end

# Add on julia_version extended tag
str = string(str, "-julia_version+", VersionNumber(VERSION.major, VERSION.minor, VERSION.patch))

if !occursin("-julia_version+", str)
str = string(str, "-julia_version+", VersionNumber(VERSION.major, VERSION.minor, VERSION.patch))
end
return str
end

Expand Down
Loading

0 comments on commit 5f43f3e

Please sign in to comment.