From 97ecdb8595c4a1fbe68ba6f39b3244e8cdabc2c6 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 28 Sep 2024 19:02:49 -0400 Subject: [PATCH] add --trim option for generating smaller binaries (#55047) This adds a command line option `--trim` that builds images where code is only included if it is statically reachable from methods marked using the new function `entrypoint`. Compile-time errors are given for call sites that are too dynamic to allow trimming the call graph (however there is an `unsafe` option if you want to try building anyway to see what happens). The PR has two other components. One is changes to Base that generally allow more code to be compiled in this mode. These changes will either be merged in separate PRs or moved to a separate part of the workflow (where we will build a custom system image for this purpose). The branch is set up this way to make it easy to check out and try the functionality. The other component is everything in the `juliac/` directory, which implements a compiler driver script based on this new option, along with some examples and tests. This will eventually become a package "app" that depends on PackageCompiler and provides a CLI for all of this stuff, so it will not be merged here. To try an example: ``` julia contrib/juliac.jl --output-exe hello --trim test/trimming/hello.jl ``` When stripped the resulting executable is currently about 900kb on my machine. Also includes a lot of work by @topolarity --------- Co-authored-by: Gabriel Baraldi Co-authored-by: Tim Holy Co-authored-by: Cody Tapscott --- Makefile | 4 +- NEWS.md | 2 + base/experimental.jl | 14 + base/libuv.jl | 5 +- base/options.jl | 1 + base/reflection.jl | 11 +- base/strings/io.jl | 4 + contrib/julia-config.jl | 2 +- contrib/juliac-buildscript.jl | 277 ++++++++++++++++++ contrib/juliac.jl | 110 +++++++ doc/src/devdocs/sysimg.md | 77 +++++ doc/src/manual/command-line-interface.md | 2 +- src/aotcompile.cpp | 69 ++++- src/cgutils.cpp | 12 + src/codegen-stubs.c | 1 + src/codegen.cpp | 355 +++++++++++++++++++++-- src/gf.c | 26 +- src/init.c | 11 +- src/jitlayers.h | 6 +- src/jl_exported_funcs.inc | 3 + src/jloptions.c | 26 +- src/jloptions.h | 1 + src/julia.expmap.in | 4 +- src/julia.h | 7 + src/julia_internal.h | 7 +- src/module.c | 2 +- src/precompile.c | 10 +- src/precompile_utils.c | 80 +++++ src/staticdata.c | 190 ++++++++++-- src/support/arraylist.h | 17 +- stdlib/LinearAlgebra/src/blas.jl | 2 +- stdlib/LinearAlgebra/src/lbt.jl | 4 +- test/Makefile | 8 +- test/trimming/Makefile | 55 ++++ test/trimming/hello.jl | 6 + test/trimming/init.c | 9 + test/trimming/trimming.jl | 7 + 37 files changed, 1338 insertions(+), 89 deletions(-) create mode 100644 contrib/juliac-buildscript.jl create mode 100644 contrib/juliac.jl create mode 100644 test/trimming/Makefile create mode 100644 test/trimming/hello.jl create mode 100644 test/trimming/init.c create mode 100644 test/trimming/trimming.jl diff --git a/Makefile b/Makefile index e440f243d876e..4fd8b878c5d1f 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac.jl $(build_datarootdir)/julia/juliac-buildscript.jl @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps @@ -181,7 +181,7 @@ $(build_sysconfdir)/julia/startup.jl: $(JULIAHOME)/etc/startup.jl | $(build_sysc @echo Creating usr/etc/julia/startup.jl @cp $< $@ -$(build_datarootdir)/julia/julia-config.jl: $(JULIAHOME)/contrib/julia-config.jl | $(build_datarootdir)/julia +$(build_datarootdir)/julia/%: $(JULIAHOME)/contrib/% | $(build_datarootdir)/julia $(INSTALL_M) $< $(dir $@) $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir) diff --git a/NEWS.md b/NEWS.md index 9ecdd87f0c2bb..ca2bf1f615012 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,8 @@ Julia v1.12 Release Notes New language features --------------------- +- New option `--trim` for building "trimmed" binaries, where code not provably reachable from entry points + is removed. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). - A new keyword argument `usings::Bool` has been added to `names`. By using this, we can now find all the names available in module `A` by `names(A; all=true, imported=true, usings=true)`. ([#54609]) - the `@atomic(...)` macro family supports now the reference assignment syntax, e.g. diff --git a/base/experimental.jl b/base/experimental.jl index 58c7258120f3f..6e757e9fa0e5f 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -457,4 +457,18 @@ without adding them to the global method table. """ :@MethodTable +""" + Base.Experimental.entrypoint(f, argtypes::Tuple) + +Mark a method for inclusion when the `--trim` option is specified. +""" +function entrypoint(@nospecialize(f), @nospecialize(argtypes::Tuple)) + entrypoint(Tuple{Core.Typeof(f), argtypes...}) +end + +function entrypoint(@nospecialize(argt::Type)) + ccall(:jl_add_entrypoint, Int32, (Any,), argt) + nothing +end + end diff --git a/base/libuv.jl b/base/libuv.jl index 143201598fde0..3c9f79dfa7b2c 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -133,7 +133,10 @@ function uv_return_spawn end function uv_asynccb end function uv_timercb end -function reinit_stdio() +reinit_stdio() = _reinit_stdio() +# we need this so it can be called by codegen to print errors, even after +# reinit_stdio has been redefined by the juliac build script. +function _reinit_stdio() global stdin = init_stdio(ccall(:jl_stdin_stream, Ptr{Cvoid}, ()))::IO global stdout = init_stdio(ccall(:jl_stdout_stream, Ptr{Cvoid}, ()))::IO global stderr = init_stdio(ccall(:jl_stderr_stream, Ptr{Cvoid}, ()))::IO diff --git a/base/options.jl b/base/options.jl index 41ce3c9e20909..1de7a2acb1e06 100644 --- a/base/options.jl +++ b/base/options.jl @@ -58,6 +58,7 @@ struct JLOptions permalloc_pkgimg::Int8 heap_size_hint::UInt64 trace_compile_timing::Int8 + trim::Int8 end # This runs early in the sysimage != is not defined yet diff --git a/base/reflection.jl b/base/reflection.jl index 5b395efc58190..fe48b6f9aa6b9 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1504,6 +1504,13 @@ struct CodegenParams """ use_jlplt::Cint + """ + If enabled, only provably reachable code (from functions marked with `entrypoint`) is included + in the output system image. Errors or warnings can be given for call sites too dynamic to handle. + The option is disabled by default. (0=>disabled, 1=>safe (static errors), 2=>unsafe, 3=>unsafe plus warnings) + """ + trim::Cint + """ A pointer of type @@ -1519,14 +1526,14 @@ struct CodegenParams prefer_specsig::Bool=false, gnu_pubnames::Bool=true, debug_info_kind::Cint = default_debug_info_kind(), debug_info_level::Cint = Cint(JLOptions().debug_level), safepoint_on_entry::Bool=true, - gcstack_arg::Bool=true, use_jlplt::Bool=true, + gcstack_arg::Bool=true, use_jlplt::Bool=true, trim::Cint=Cint(0), lookup::Ptr{Cvoid}=unsafe_load(cglobal(:jl_rettype_inferred_addr, Ptr{Cvoid}))) return new( Cint(track_allocations), Cint(code_coverage), Cint(prefer_specsig), Cint(gnu_pubnames), debug_info_kind, debug_info_level, Cint(safepoint_on_entry), - Cint(gcstack_arg), Cint(use_jlplt), + Cint(gcstack_arg), Cint(use_jlplt), Cint(trim), lookup) end end diff --git a/base/strings/io.jl b/base/strings/io.jl index 754e058cd2f54..df34712b519d5 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -51,6 +51,8 @@ function print(io::IO, xs...) return nothing end +setfield!(typeof(print).name.mt, :max_args, 10, :monotonic) + """ println([io::IO], xs...) @@ -74,6 +76,7 @@ julia> String(take!(io)) """ println(io::IO, xs...) = print(io, xs..., "\n") +setfield!(typeof(println).name.mt, :max_args, 10, :monotonic) ## conversion of general objects to strings ## """ @@ -149,6 +152,7 @@ function print_to_string(xs...) end String(_unsafe_take!(s)) end +setfield!(typeof(print_to_string).name.mt, :max_args, 10, :monotonic) function string_with_env(env, xs...) if isempty(xs) diff --git a/contrib/julia-config.jl b/contrib/julia-config.jl index c692b3f522fb2..8b1eb55cbe4f4 100755 --- a/contrib/julia-config.jl +++ b/contrib/julia-config.jl @@ -67,7 +67,7 @@ function ldlibs(doframework) "julia" end if Sys.isunix() - return "-Wl,-rpath,$(shell_escape(libDir())) -Wl,-rpath,$(shell_escape(private_libDir())) -l$libname" + return "-L$(shell_escape(private_libDir())) -Wl,-rpath,$(shell_escape(libDir())) -Wl,-rpath,$(shell_escape(private_libDir())) -l$libname" else return "-l$libname -lopenlibm" end diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl new file mode 100644 index 0000000000000..50f96198c416b --- /dev/null +++ b/contrib/juliac-buildscript.jl @@ -0,0 +1,277 @@ +# Script to run in the process that generates juliac's object file output + +inputfile = ARGS[1] +output_type = ARGS[2] +add_ccallables = ARGS[3] == "true" + +# Initialize some things not usually initialized when output is requested +Sys.__init__() +Base.init_depot_path() +Base.init_load_path() +Base.init_active_project() +task = current_task() +task.rngState0 = 0x5156087469e170ab +task.rngState1 = 0x7431eaead385992c +task.rngState2 = 0x503e1d32781c2608 +task.rngState3 = 0x3a77f7189200c20b +task.rngState4 = 0x5502376d099035ae +uuid_tuple = (UInt64(0), UInt64(0)) +ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) +ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred) + +# Patch methods in Core and Base + +@eval Core begin + DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) +end + +(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) + +@eval Base begin + _assert_tostring(msg) = "" + reinit_stdio() = nothing + JuliaSyntax.enable_in_core!() = nothing + init_active_project() = ACTIVE_PROJECT[] = nothing + set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile + disable_library_threading() = nothing + start_profile_listener() = nothing + @inline function invokelatest(f::F, args...; kwargs...) where F + return f(args...; kwargs...) + end + function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} + s = IOBuffer(sizehint=sizehint) + if context isa Tuple + f(IOContext(s, context...), args...) + elseif context !== nothing + f(IOContext(s, context), args...) + else + f(s, args...) + end + String(_unsafe_take!(s)) + end + function show_typeish(io::IO, @nospecialize(T)) + if T isa Type + show(io, T) + elseif T isa TypeVar + print(io, (T::TypeVar).name) + else + print(io, "?") + end + end + function show(io::IO, T::Type) + if T isa DataType + print(io, T.name.name) + if T !== T.name.wrapper && length(T.parameters) > 0 + print(io, "{") + first = true + for p in T.parameters + if !first + print(io, ", ") + end + first = false + if p isa Int + show(io, p) + elseif p isa Type + show(io, p) + elseif p isa Symbol + print(io, ":") + print(io, p) + elseif p isa TypeVar + print(io, p.name) + else + print(io, "?") + end + end + print(io, "}") + end + elseif T isa Union + print(io, "Union{") + show_typeish(io, T.a) + print(io, ", ") + show_typeish(io, T.b) + print(io, "}") + elseif T isa UnionAll + print(io, T.body::Type) + print(io, " where ") + print(io, T.var.name) + end + end + show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) + + mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) where {F, F2} = + _mapreduce_dim(f, op, init, A, dims) + mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted...; kw...) where {F, F2} = + reduce(op, map(f, A...); kw...) + + _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = + mapfoldl_impl(f, op, nt, A) + + _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = + _mapreduce(f, op, IndexStyle(A), A) + + _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, dims) where {F, F2} = + mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) + + _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) where {F,F2} = + mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) + + mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F, F2} = + reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) + mapreduce_first(f::F, op::F2, x) where {F,F2} = reduce_first(op, f(x)) + + _mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted) where {F,F2} = _mapreduce(f, op, IndexStyle(A), A) + mapreduce_empty(::typeof(identity), op::F, T) where {F} = reduce_empty(op, T) + mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) + mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) +end +@eval Base.Unicode begin + function utf8proc_map(str::Union{String,SubString{String}}, options::Integer, chartransform::F = identity) where F + nwords = utf8proc_decompose(str, options, C_NULL, 0, chartransform) + buffer = Base.StringVector(nwords*4) + nwords = utf8proc_decompose(str, options, buffer, nwords, chartransform) + nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options) + nbytes < 0 && utf8proc_error(nbytes) + return String(resize!(buffer, nbytes)) + end +end +@eval Base.GMP begin + function __init__() + try + ccall((:__gmp_set_memory_functions, libgmp), Cvoid, + (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), + cglobal(:jl_gc_counted_malloc), + cglobal(:jl_gc_counted_realloc_with_old_size), + cglobal(:jl_gc_counted_free_with_size)) + ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL + ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) + catch ex + Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") + end + # This only works with a patched version of GMP, ignore otherwise + try + ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, + (Ptr{Cvoid},), + cglobal(:jl_throw_out_of_memory_error)) + ALLOC_OVERFLOW_FUNCTION[] = true + catch ex + # ErrorException("ccall: could not find function...") + if typeof(ex) != ErrorException + rethrow() + end + end + end +end +@eval Base.Sort begin + issorted(itr; + lt::T=isless, by::F=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) where {T,F} = + issorted(itr, ord(lt,by,rev,order)) +end +@eval Base.TOML begin + function try_return_datetime(p, year, month, day, h, m, s, ms) + return DateTime(year, month, day, h, m, s, ms) + end + function try_return_date(p, year, month, day) + return Date(year, month, day) + end + function parse_local_time(l::Parser) + h = @try parse_int(l, false) + h in 0:23 || return ParserError(ErrParsingDateTime) + _, m, s, ms = @try _parse_local_time(l, true) + # TODO: Could potentially parse greater accuracy for the + # fractional seconds here. + return try_return_time(l, h, m, s, ms) + end + function try_return_time(p, h, m, s, ms) + return Time(h, m, s, ms) + end +end + +# Load user code + +import Base.Experimental.entrypoint + +let mod = Base.include(Base.__toplevel__, inputfile) + if !isa(mod, Module) + mod = Main + end + if output_type == "--output-exe" && isdefined(mod, :main) && !add_ccallables + entrypoint(mod.main, ()) + end + #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String)) + #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) + entrypoint(Base.task_done_hook, (Task,)) + entrypoint(Base.wait, ()) + entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) + entrypoint(Base.checktaskempty, ()) + if add_ccallables + ccall(:jl_add_ccallable_entrypoints, Cvoid, ()) + end +end + +# Additional method patches depending on whether user code loads certain stdlibs + +let loaded = Symbol.(Base.loaded_modules_array()) # TODO better way to do this + if :SparseArrays in loaded + using SparseArrays + @eval SparseArrays.CHOLMOD begin + function __init__() + ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) + end + end + end + if :Artifacts in loaded + using Artifacts + @eval Artifacts begin + function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, _::Val{lazyartifacts}) where lazyartifacts + moduleroot = Base.moduleroot(__module__) + if haskey(Base.module_keys, moduleroot) + # Process overrides for this UUID, if we know what it is + process_overrides(artifact_dict, Base.module_keys[moduleroot].uuid) + end + + # If the artifact exists, we're in the happy path and we can immediately + # return the path to the artifact: + dirs = artifact_paths(hash; honor_overrides=true) + for dir in dirs + if isdir(dir) + return jointail(dir, path_tail) + end + end + end + end + end + if :Pkg in loaded + using Pkg + @eval Pkg begin + __init__() = rand() #TODO, methods that do nothing don't get codegened + end + end + if :StyledStrings in loaded + using StyledStrings + @eval StyledStrings begin + __init__() = rand() + end + end +end + +empty!(Core.ARGS) +empty!(Base.ARGS) +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +empty!(Base.TOML_CACHE.d) +Base.TOML.reinit!(Base.TOML_CACHE.p, "") +Base.ACTIVE_PROJECT[] = nothing +@eval Base begin + PROGRAM_FILE = "" +end +@eval Sys begin + BINDIR = "" + STDLIB = "" +end diff --git a/contrib/juliac.jl b/contrib/juliac.jl new file mode 100644 index 0000000000000..61e0e91958667 --- /dev/null +++ b/contrib/juliac.jl @@ -0,0 +1,110 @@ +# Julia compiler wrapper script +# NOTE: The interface and location of this script are considered unstable/experimental + +cmd = Base.julia_cmd() +cmd = `$cmd --startup-file=no --history-file=no` +output_type = nothing # exe, sharedlib, sysimage +trim = nothing +outname = nothing +file = nothing +add_ccallables = false + +help = findfirst(x->x == "--help", ARGS) +if help !== nothing + println( + """ + Usage: julia juliac.jl [--output-exe | --output-lib | --output-sysimage] [options] + --trim= Only output code statically determined to be reachable + --compile-ccallable Include all methods marked `@ccallable` in output + --verbose Request verbose output + """) + exit(0) +end + +let i = 1 + while i <= length(ARGS) + arg = ARGS[i] + if arg == "--output-exe" || arg == "--output-lib" || arg == "--output-sysimage" + isnothing(output_type) || error("Multiple output types specified") + global output_type = arg + i == length(ARGS) && error("Output specifier requires an argument") + global outname = ARGS[i+1] + i += 1 + elseif startswith(arg, "--trim") + arg = split(arg, '=') + if length(arg) == 1 + global trim = "safe" + else + global trim = arg[2] + end + elseif arg == "--compile-ccallable" + global add_ccallables = true + else + if arg[1] == '-' || !isnothing(file) + println("Unexpected argument `$arg`") + exit(1) + end + global file = arg + end + i += 1 + end +end + +isnothing(outname) && error("No output file specified") +isnothing(file) && error("No input file specified") + +absfile = abspath(file) +cflags = readchomp(`$(cmd) $(joinpath(Sys.BINDIR, Base.DATAROOTDIR,"julia", "julia-config.jl")) --cflags `) +cflags = Base.shell_split(cflags) +allflags = readchomp(`$(cmd) $(joinpath(Sys.BINDIR, Base.DATAROOTDIR,"julia", "julia-config.jl")) --allflags`) +allflags = Base.shell_split(allflags) +tmpdir = mktempdir(cleanup=false) +initsrc_path = joinpath(tmpdir, "init.c") +init_path = joinpath(tmpdir, "init.a") +img_path = joinpath(tmpdir, "img.a") +bc_path = joinpath(tmpdir, "img-bc.a") + +open(initsrc_path, "w") do io + print(io, """ + #include + __attribute__((constructor)) void static_init(void) { + if (jl_is_initialized()) + return; + julia_init(JL_IMAGE_IN_MEMORY); + jl_exception_clear(); + } + """) +end + +static_call_graph_arg() = isnothing(trim) ? `` : `--trim=$(trim)` +is_verbose() = verbose ? `--verbose-compilation=yes` : `` +cmd = addenv(`$cmd --project=$(Base.active_project()) --output-o $img_path --output-incremental=no --strip-ir --strip-metadata $(static_call_graph_arg()) $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + +if !success(pipeline(cmd; stdout, stderr)) + println(stderr, "\nFailed to compile $file") + exit(1) +end + +run(`cc $(cflags) -g -c -o $init_path $initsrc_path`) + +if output_type == "--output-lib" || output_type == "--output-sysimage" + of, ext = splitext(outname) + soext = "." * Base.BinaryPlatforms.platform_dlext() + if ext == "" + outname = of * soext + end +end + +julia_libs = Base.shell_split(Base.isdebugbuild() ? "-ljulia-debug -ljulia-internal-debug" : "-ljulia -ljulia-internal") +try + if output_type == "--output-lib" + run(`cc $(allflags) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)`) + elseif output_type == "--output-sysimage" + run(`cc $(allflags) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)`) + else + run(`cc $(allflags) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)`) + end +catch + println("\nCompilation failed.") + exit(1) +end diff --git a/doc/src/devdocs/sysimg.md b/doc/src/devdocs/sysimg.md index 7d4f7afdbb86a..64c309e1fb02a 100644 --- a/doc/src/devdocs/sysimg.md +++ b/doc/src/devdocs/sysimg.md @@ -117,3 +117,80 @@ See code comments for each components for more implementation details. depending on the ISA. The target selection will prefer exact CPU name match, larger vector register size, and larger number of features. An overview of this process is in `src/processor.cpp`. + +## Trimming + +System images are typically quite large, since Base includes a lot of functionality, and by +default system images also include several packages such as LinearAlgebra for convenience +and backwards compatibility. Most programs will use only a fraction of the functions in +these packages. Therefore it makes sense to build binaries that exclude unused functions +to save space, referred to as "trimming". + +While the basic idea of trimming is sound, Julia has dynamic and reflective features that make it +difficult (or impossible) to know in general which functions are unused. As an extreme example, +consider code like + +``` +getglobal(Base, Symbol(readchomp(stdin)))(1) +``` + +This code reads a function name from `stdin` and calls the named function from Base on the value +`1`. In this case it is impossible to predict which function will be called, so no functions +can reliably be considered "unused". With some noteworthy exceptions (Julia's own REPL being +one of them), most real-world programs do not do things like this. + +Less extreme cases occur, for example, when there are type instabilities that make it impossible +for the compiler to predict which method will be called. However, if code is well-typed and does +not use reflection, a complete and (hopefully) relatively small set of needed methods can be +determined, and the rest can be removed. The `--trim` command-line option requests this kind of +compilation. + +When `--trim` is specified in a command used to build a system image, the compiler begins +tracing calls starting at methods marked using `Base.Experimental.entrypoint`. If a call is too +dynamic to reasonably narrow down the possible call targets, an error is given at compile +time showing the location of the call. For testing purposes, it is possible to skip these +errors by specifying `--trim=unsafe` or `--trim=unsafe-warn`. Then you will get a system +image built, but it may crash at run time if needed code is not present. + +It typically makes sense to specify `--strip-ir` along with `--trim`, since trimmed binaries +are fully compiled and therefore don't need Julia IR. At some point we may make `--trim` imply +`--strip-ir`, but for now we have kept them orthogonal. + +To get the smallest possible binary, it will also help to specify `--strip-metadata` and +run the Unix `strip` utility. However, those steps remove Julia-specific and native (DWARF format) +debug info, respectively, and so will make debugging more difficult. + +### Common problems + +- The Base global variables `stdin`, `stdout`, and `stderr` are non-constant and so their + types are not known. All printing should use a specific IO object with a known type. + The easiest substitution is to use `print(Core.stdout, x)` instead of `print(x)` or + `print(stdout, x)`. +- Use tools like `JET`, `Cthulhu`, and/or `SnoopCompile` to identify failures of type-inference, and + follow our [Performance Tips](@ref) to fix them. + +### Compatibility concerns + +We have identified many small changes to Base that significantly increase the set of programs +that can be reliably trimmed. Unfortunately some of those changes would be considered breaking, +and so are only applied when trimming is requested (this is done by an external build script, +currently maintained inside the test suite as `test/trimming/buildscript.jl`). +Therefore in many cases trimming will require you to opt in to new variants of Base and some +standard libraries. + +If you want to use trimming, it is important to set up continuous integration testing that +performs a trimmed build and fully tests the resulting program. +Fortunately, if your program successfully compiles with `--trim` then it is very likely to work +the same as it did before. However, CI is needed to ensure that your program continues to build +with trimming as you develop it. + +Package authors may wish to test that their package is "trimming safe", however this is impossible +in general. Trimming is only expected to work given concrete entry points such as `main()` and +library entry points meant to be called from outside Julia. For generic packages, existing tests +for type stability like `@inferred` and `JET` are about as close as you can get to checking +trim compatibility. + +Trimming also introduces new compatibility issues between minor versions of Julia. At this time, +we are not able to guarantee that a program that can be trimmed in one version of Julia +can also be trimmed in all future versions of Julia. However, breakage of that kind is expected +to be rare. We also plan to try to *increase* the set of programs that can be trimmed over time. diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 41c3eacd61d26..ef20e51ea6e4e 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -218,7 +218,7 @@ The following is a complete list of command-line switches available when launchi |`--trace-compile-timing` |If --trace-compile is enabled show how long each took to compile in ms| |`--image-codegen` |Force generate code in imaging mode| |`--permalloc-pkgimg={yes\|no*}` |Copy the data section of package images into memory| - +|`--trim={no*|safe|unsafe|unsafe-warn}` |Build a sysimage including only code provably reachable from methods marked by calling `entrypoint`. The three non-default options differ in how they handle dynamic call sites. In safe mode, such sites result in compile-time errors. In unsafe mode, such sites are allowed but the resulting binary might be missing needed code and can throw runtime errors. With unsafe-warn, such sites will trigger warnings at compile-time and might error at runtime.| !!! compat "Julia 1.1" In Julia 1.0, the default `--project=@.` option did not search up from the root diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index b4c8ef6095a55..c2f112f9c9d5c 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -95,6 +95,17 @@ void jl_get_function_id_impl(void *native_code, jl_code_instance_t *codeinst, } } +extern "C" JL_DLLEXPORT_CODEGEN +void jl_get_llvm_mis_impl(void *native_code, arraylist_t* MIs) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + auto map = data->jl_fvar_map; + for (auto &ci : map) { + jl_method_instance_t *mi = ci.first->def; + arraylist_push(MIs, mi); + } +} + extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_gvs_impl(void *native_code, arraylist_t *gvs) { @@ -284,6 +295,7 @@ jl_code_instance_t *jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_ jl_value_t *ci = cgparams.lookup(mi, world, world); JL_GC_PROMISE_ROOTED(ci); jl_code_instance_t *codeinst = NULL; + JL_GC_PUSH1(&codeinst); if (ci != jl_nothing && jl_atomic_load_relaxed(&((jl_code_instance_t *)ci)->inferred) != jl_nothing) { codeinst = (jl_code_instance_t*)ci; } @@ -301,9 +313,11 @@ jl_code_instance_t *jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_ jl_mi_cache_insert(mi, codeinst); } } + JL_GC_POP(); return codeinst; } +arraylist_t new_invokes; // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup, and can // also be used be extern consumers like GPUCompiler.jl to obtain a module containing @@ -353,8 +367,12 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm params.imaging_mode = imaging; params.debug_level = cgparams->debug_info_level; params.external_linkage = _external_linkage; + arraylist_new(&new_invokes, 0); size_t compile_for[] = { jl_typeinf_world, _world }; - for (int worlds = 0; worlds < 2; worlds++) { + int worlds = 0; + if (jl_options.trim != JL_TRIM_NO) + worlds = 1; + for (; worlds < 2; worlds++) { JL_TIMING(NATIVE_AOT, NATIVE_Codegen); size_t this_world = compile_for[worlds]; if (!this_world) @@ -373,6 +391,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm continue; } mi = (jl_method_instance_t*)item; +compile_mi: src = NULL; // if this method is generally visible to the current compilation world, // and this is either the primary world, or not applicable in the primary world @@ -380,16 +399,47 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm if (jl_atomic_load_relaxed(&mi->def.method->primary_world) <= this_world && this_world <= jl_atomic_load_relaxed(&mi->def.method->deleted_world)) { // find and prepare the source code to compile jl_code_instance_t *codeinst = jl_ci_cache_lookup(*cgparams, mi, this_world); - if (codeinst && !params.compiled_functions.count(codeinst)) { + if (jl_options.trim != JL_TRIM_NO && !codeinst) { + // If we're building a small image, we need to compile everything + // to ensure that we have all the information we need. + jl_safe_printf("Codegen decided not to compile code root"); + jl_(mi); + abort(); + } + if (codeinst && !params.compiled_functions.count(codeinst) && !data->jl_fvar_map.count(codeinst)) { // now add it to our compilation results - JL_GC_PROMISE_ROOTED(codeinst->rettype); - orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(codeinst->def), - params.tsctx, clone.getModuleUnlocked()->getDataLayout(), - Triple(clone.getModuleUnlocked()->getTargetTriple())); - jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, NULL, params); - if (result_m) - params.compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; + // Const returns do not do codegen, but juliac inspects codegen results so make a dummy fvar entry to represent it + if (jl_options.trim != JL_TRIM_NO && jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) { + data->jl_fvar_map[codeinst] = std::make_tuple((uint32_t)-3, (uint32_t)-3); + } else { + JL_GC_PROMISE_ROOTED(codeinst->rettype); + orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(codeinst->def), + params.tsctx, clone.getModuleUnlocked()->getDataLayout(), + Triple(clone.getModuleUnlocked()->getTargetTriple())); + jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, NULL, params); + if (result_m) + params.compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; + else if (jl_options.trim != JL_TRIM_NO) { + // if we're building a small image, we need to compile everything + // to ensure that we have all the information we need. + jl_safe_printf("codegen failed to compile code root"); + jl_(mi); + abort(); + } + } } + } else if (this_world != jl_typeinf_world) { + /* + jl_safe_printf("Codegen could not find requested codeinstance to be compiled\n"); + jl_(mi); + abort(); + */ + } + // TODO: is goto the best way to do this? + jl_compile_workqueue(params, policy); + mi = (jl_method_instance_t*)arraylist_pop(&new_invokes); + if (mi != NULL) { + goto compile_mi; } } @@ -397,6 +447,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm jl_compile_workqueue(params, policy); } JL_GC_POP(); + arraylist_free(&new_invokes); // process the globals array, before jl_merge_module destroys them SmallVector gvars(params.global_targets.size()); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 7f96bb1047abc..4547e693755cd 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2336,6 +2336,12 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { + if (trim_may_error(ctx.params->trim)) { + // if we know the return type, we can assume the result is of that type + errs() << "ERROR: Dynamic call to setfield/modifyfield\n"; + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); ret = mark_julia_type(ctx, callval, true, jl_any_type); } @@ -4077,6 +4083,12 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { + if (trim_may_error(ctx.params->trim)) { + // if we know the return type, we can assume the result is of that type + errs() << "ERROR: Dynamic call to setfield/modifyfield\n"; + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); rhs = mark_julia_type(ctx, callval, true, jl_any_type); } diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 41812d903816c..7ddb68fd6b036 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -15,6 +15,7 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, ios_t *z, ios_t *s) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE +JL_DLLEXPORT void jl_get_llvm_mis_fallback(void *native_code, arraylist_t* MIs) UNAVAILABLE JL_DLLEXPORT void jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_method_asm_fallback(jl_method_instance_t *linfo, size_t world, diff --git a/src/codegen.cpp b/src/codegen.cpp index a452e0fccd0c5..a7a985284c87b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include // target machine computation @@ -1651,31 +1652,23 @@ static const auto &builtin_func_map() { return builtins; } +static const auto &may_dispatch_builtins() { + static std::unordered_set builtins( + {jl_f__apply_iterate_addr, + jl_f__apply_pure_addr, + jl_f__call_in_world_addr, + jl_f__call_in_world_total_addr, + jl_f__call_latest_addr, + }); + return builtins; +} + static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; static _Atomic(uint64_t) globalUniqueGeneratedNames{1}; // --- code generation --- -extern "C" { - jl_cgparams_t jl_default_cgparams = { - /* track_allocations */ 1, - /* code_coverage */ 1, - /* prefer_specsig */ 0, -#ifdef _OS_WINDOWS_ - /* gnu_pubnames */ 0, -#else - /* gnu_pubnames */ 1, -#endif - /* debug_info_kind */ (int) DICompileUnit::DebugEmissionKind::FullDebug, - /* debug_line_info */ 1, - /* safepoint_on_entry */ 1, - /* gcstack_arg */ 1, - /* use_jlplt*/ 1, - /* lookup */ jl_rettype_inferred_addr }; -} - - static MDNode *best_tbaa(jl_tbaacache_t &tbaa_cache, jl_value_t *jt) { jt = jl_unwrap_unionall(jt); if (jt == (jl_value_t*)jl_datatype_type || @@ -1987,7 +1980,7 @@ class jl_codectx_t { size_t max_world = -1; const char *name = NULL; StringRef file{}; - ssize_t *line = NULL; + int32_t line = -1; Value *spvals_ptr = NULL; Value *argArray = NULL; Value *argCount = NULL; @@ -2146,6 +2139,179 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); static void recombine_value(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dst, jl_aliasinfo_t const &dst_ai, Align alignment, bool isVolatile); +static void print_stack_crumbs(jl_codectx_t &ctx) +{ + errs() << "\n"; + errs() << "Stacktrace:\n"; + jl_method_instance_t *caller = ctx.linfo; + jl_((jl_value_t*)caller); + errs() << "In " << ctx.file << ":" << ctx.line << "\n"; + while (true) { + auto it = ctx.emission_context.enqueuers.find(caller); + if (it != ctx.emission_context.enqueuers.end()) { + caller = std::get(it->second); + } else { + break; + } + if (caller) { + if (jl_is_method_instance(caller)) { + for (auto it2 = std::get(it->second).begin(); it2 != (std::prev(std::get(it->second).end())); ++it2) { + auto frame = *it2; + errs() << std::get<0>(frame) << " \n"; + errs() << "In " << std::get<1>(frame) << ":" << std::get(frame) << "\n"; + } + auto &frame = std::get(it->second).front(); + jl_((jl_value_t*)caller); + errs() << "In " << std::get<1>(frame) << ":" << std::get(frame) << "\n"; + } + } + else + break; + } + abort(); +} + +static jl_value_t *StackFrame( + jl_value_t *linfo, + std::string fn_name, + std::string filepath, + int32_t lineno, + jl_value_t *inlined) +{ + jl_value_t *StackFrame = jl_get_global(jl_base_module, jl_symbol("StackFrame")); + assert(StackFrame != nullptr); + + jl_value_t *args[7] = { + /* func */ (jl_value_t *)jl_symbol(fn_name.c_str()), + /* line */ (jl_value_t *)jl_symbol(filepath.c_str()), + /* line */ jl_box_int32(lineno), + /* linfo */ (jl_value_t *)linfo, + /* from_c */ jl_false, + /* inlined */ inlined, + /* pointer */ jl_box_uint64(0) + }; + + jl_value_t *frame = nullptr; + JL_TRY { + frame = jl_apply_generic(StackFrame, args, 7); + } JL_CATCH { + jl_safe_printf("Error creating stack frame\n"); + } + return frame; +} + +static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee, int no_debug=false) +{ + CallFrames frames; + auto it = ctx.emission_context.enqueuers.find(callee); + if (it != ctx.emission_context.enqueuers.end()) + return; + if (no_debug) { // Used in tojlinvoke + frames.push_back({ctx.funcName, "", 0}); + ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); + return; + } + auto DL = ctx.builder.getCurrentDebugLocation(); + auto filename = std::string(DL->getFilename()); + auto line = DL->getLine(); + auto fname = std::string(DL->getScope()->getSubprogram()->getName()); + frames.push_back({fname, filename, line}); + auto DI = DL.getInlinedAt(); + while (DI) { + auto filename = std::string(DI->getFilename()); + auto line = DI->getLine(); + auto fname = std::string(DI->getScope()->getSubprogram()->getName()); + frames.push_back({fname, filename, line}); + DI = DI->getInlinedAt(); + } + ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); +} + +static jl_array_t* build_stack_crumbs(jl_codectx_t &ctx) JL_NOTSAFEPOINT +{ + static intptr_t counter = 5; + jl_method_instance_t *caller = (jl_method_instance_t*)counter; //nothing serves as a sentinel for the bottom for the stack + push_frames(ctx, ctx.linfo, (jl_method_instance_t*)caller); + counter++; + jl_array_t *out = jl_alloc_array_1d(jl_array_any_type, 0); + JL_GC_PUSH1(&out); + while (true) { + auto it = ctx.emission_context.enqueuers.find(caller); + if (it != ctx.emission_context.enqueuers.end()) { + caller = std::get(it->second); + } else { + break; + } + if (caller) { + assert(ctx.emission_context.enqueuers.count(caller) == 1); + if (jl_is_method_instance(caller)) { + //TODO: Use a subrange when C++20 is a thing + for (auto it2 = std::get(it->second).begin(); it2 != (std::prev(std::get(it->second).end())); ++it2) { + auto frame = *it2; + jl_value_t *stackframe = StackFrame(jl_nothing, std::get<0>(frame), std::get<1>(frame), std::get(frame), jl_true); + if (stackframe == nullptr) + print_stack_crumbs(ctx); + jl_array_ptr_1d_push(out, stackframe); + } + auto &frame = std::get(it->second).back(); + jl_value_t *stackframe = StackFrame((jl_value_t *)caller, std::get<0>(frame), std::get<1>(frame), std::get(frame), jl_false); + if (stackframe == nullptr) + print_stack_crumbs(ctx); + jl_array_ptr_1d_push(out, stackframe); + } + } + else + break; + } + JL_GC_POP(); + return out; +} + +static void print_stacktrace(jl_codectx_t &ctx, int trim) +{ + jl_task_t *ct = jl_get_current_task(); + assert(ct); + + // Temporarily operate in the current age + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); + jl_array_t* bt = build_stack_crumbs(ctx); + JL_GC_PUSH1(&bt); + + // Call `reinit_stdio` to get TTY IO objects (w/ color) + jl_value_t *reinit_stdio = jl_get_global(jl_base_module, jl_symbol("_reinit_stdio")); + assert(reinit_stdio); + jl_apply_generic(reinit_stdio, nullptr, 0); + + // Show the backtrace + jl_value_t *show_backtrace = jl_get_global(jl_base_module, jl_symbol("show_backtrace")); + jl_value_t *base_stderr = jl_get_global(jl_base_module, jl_symbol("stderr")); + assert(show_backtrace && base_stderr); + + JL_TRY { + jl_value_t *args[2] = { base_stderr, (jl_value_t *)bt }; + jl_apply_generic(show_backtrace, args, 2); + } JL_CATCH { + jl_printf(JL_STDERR,"Error showing backtrace\n"); + print_stack_crumbs(ctx); + } + + jl_printf(JL_STDERR, "\n\n"); + JL_GC_POP(); + ct->world_age = last_age; + + if (trim == JL_TRIM_SAFE) { + jl_printf(JL_STDERR,"Aborting compilation due to finding a dynamic dispatch"); + exit(1); + } + return; +} + +static int trim_may_error(int trim) +{ + return (trim == JL_TRIM_SAFE) || (trim == JL_TRIM_UNSAFE_WARN); +} + static GlobalVariable *prepare_global_in(Module *M, JuliaVariable *G) { return G->realize(M); @@ -4281,6 +4447,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *theArgs = emit_ptrgep(ctx, ctx.argArray, ctx.nReqArgs * sizeof(jl_value_t*)); Value *r = ctx.builder.CreateCall(prepare_call(jlapplygeneric_func), { theF, theArgs, nva }); *ret = mark_julia_type(ctx, r, true, jl_any_type); + if (trim_may_error(ctx.params->trim)) { + // if we know the return type, we can assume the result is of that type + errs() << "ERROR: Dynamic call to Core._apply_iterate detected\n"; + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } return true; } } @@ -5388,12 +5560,25 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, specsig}; + if (trim_may_error(ctx.params->trim)) + push_frames(ctx, ctx.linfo, mi); } } } } } if (!handled) { + if (trim_may_error(ctx.params->trim)) { + if (lival.constant) { + arraylist_push(&new_invokes, lival.constant); + push_frames(ctx, ctx.linfo, (jl_method_instance_t*)lival.constant); + } else { + errs() << "Dynamic call to unknown function"; + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + + print_stacktrace(ctx, ctx.params->trim); + } + } Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); result = mark_julia_type(ctx, r, true, rt); } @@ -5453,7 +5638,12 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ return mark_julia_type(ctx, oldnew, true, rt); } } - + if (trim_may_error(ctx.params->trim)) { + errs() << "ERROR: dynamic invoke modify call to"; + jl_(args[0]); + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } // emit function and arguments Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call); return mark_julia_type(ctx, callval, true, rt); @@ -5523,10 +5713,15 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); if (handled) return result; - + jl_fptr_args_t builtin_fptr = jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(f.constant)); // special case for some known builtin not handled by emit_builtin_call - auto it = builtin_func_map().find(jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(f.constant))); + auto it = builtin_func_map().find(builtin_fptr); if (it != builtin_func_map().end()) { + if (trim_may_error(ctx.params->trim) && may_dispatch_builtins().count(builtin_fptr)) { + errs() << "ERROR: Dynamic call to builtin" << jl_symbol_name(((jl_datatype_t*)jl_typeof(f.constant))->name->name); + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); setName(ctx.emission_context, ret, it->second->name + "_ret"); return mark_julia_type(ctx, ret, true, rt); @@ -5565,7 +5760,79 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo } } } + int failed_dispatch = !argv[0].constant; + if (ctx.params->trim != JL_TRIM_NO) { + size_t min_valid = 1; + size_t max_valid = ~(size_t)0; + size_t latest_world = jl_get_world_counter(); // TODO: marshal the world age of the compilation here. + + // Find all methods matching the call signature + jl_array_t *matches = NULL; + jl_value_t *tup = NULL; + JL_GC_PUSH2(&tup, &matches); + if (!failed_dispatch) { + SmallVector argtypes; + for (auto& arg: argv) + argtypes.push_back(arg.typ); + tup = jl_apply_tuple_type_v(argtypes.data(), argtypes.size()); + matches = (jl_array_t*)jl_matching_methods((jl_tupletype_t*)tup, jl_nothing, 10 /*TODO: make global*/, 1, + latest_world, &min_valid, &max_valid, NULL); + if ((jl_value_t*)matches == jl_nothing) + failed_dispatch = 1; + } + + // Expand each matching method to its unique specialization, if it has exactly one + if (!failed_dispatch) { + size_t k; + size_t len = new_invokes.len; + for (k = 0; k < jl_array_nrows(matches); k++) { + jl_method_match_t *match = (jl_method_match_t *)jl_array_ptr_ref(matches, k); + jl_method_instance_t *mi = jl_method_match_to_mi(match, latest_world, min_valid, max_valid, 0); + if (!mi) { + if (jl_array_nrows(matches) == 1) { + // if the method match is not compileable, but there is only one, fall back to + // unspecialized implementation + mi = jl_get_unspecialized(match->method); + } + else { + new_invokes.len = len; + failed_dispatch = 1; + break; + } + } + arraylist_push(&new_invokes, mi); + } + } + JL_GC_POP(); + } + if (failed_dispatch && trim_may_error(ctx.params->trim)) { + errs() << "Dynamic call to "; + jl_jmp_buf *old_buf = jl_get_safe_restore(); + jl_jmp_buf buf; + jl_set_safe_restore(&buf); + if (!jl_setjmp(buf, 0)) { + jl_static_show((JL_STREAM*)STDERR_FILENO, (jl_value_t*)args[0]); + jl_printf((JL_STREAM*)STDERR_FILENO,"("); + for (size_t i = 1; i < nargs; ++i) { + jl_value_t *typ = argv[i].typ; + if (!jl_is_concrete_type(typ)) // Print type in red + jl_printf((JL_STREAM*)STDERR_FILENO, "\x1b[31m"); + jl_static_show((JL_STREAM*)STDERR_FILENO, (jl_value_t*)argv[i].typ); + if (!jl_is_concrete_type(typ)) + jl_printf((JL_STREAM*)STDERR_FILENO, "\x1b[0m"); + if (i != nargs-1) + jl_printf((JL_STREAM*)STDERR_FILENO,", "); + } + jl_printf((JL_STREAM*)STDERR_FILENO,")\n"); + } + else { + jl_printf((JL_STREAM*)STDERR_FILENO, "\n!!! ERROR while printing error -- ABORTING !!!\n"); + } + jl_set_safe_restore(old_buf); + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } // emit function and arguments Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, n_generic_args, julia_call); return mark_julia_type(ctx, callval, true, rt); @@ -6710,6 +6977,13 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ ((jl_method_t*)source.constant)->nargs > 0 && jl_is_valid_oc_argtype((jl_tupletype_t*)argt.constant, (jl_method_t*)source.constant); + if (!can_optimize && trim_may_error(ctx.params->trim)) { + // if we know the return type, we can assume the result is of that type + errs() << "ERROR: Dynamic call to OpaqueClosure method\n"; + errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; + print_stacktrace(ctx, ctx.params->trim); + } + if (can_optimize) { jl_value_t *closure_t = NULL; jl_value_t *env_t = NULL; @@ -6909,6 +7183,11 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptr GlobalVariable::InternalLinkage, name, M); jl_init_function(f, params.TargetTriple); + if (trim_may_error(params.params->trim)) { + arraylist_push(&new_invokes, codeinst->def); // Try t compile this invoke + // TODO: Debuginfo! + push_frames(ctx, ctx.linfo, codeinst->def, 1); + } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); ctx.f = f; // for jl_Module @@ -8126,6 +8405,7 @@ static jl_llvm_functions_t if (lam && jl_is_method(lam->def.method)) { toplineno = lam->def.method->line; ctx.file = jl_symbol_name(lam->def.method->file); + ctx.line = lam->def.method->line; } else if ((jl_value_t*)src->debuginfo != jl_nothing) { // look for the file and line info of the original start of this block, as reported by lowering @@ -8134,6 +8414,7 @@ static jl_llvm_functions_t debuginfo = debuginfo->linetable; ctx.file = jl_debuginfo_file(debuginfo); struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, 0); + ctx.line = lineidx.line; toplineno = std::max((int32_t)0, lineidx.line); } if (ctx.file.empty()) @@ -9904,7 +10185,7 @@ void jl_compile_workqueue( if (it == params.compiled_functions.end()) { // Reinfer the function. The JIT came along and removed the inferred // method body. See #34993 - if (policy != CompilationPolicy::Default && + if ((policy != CompilationPolicy::Default || params.params->trim) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { // XXX: SOURCE_MODE_FORCE_SOURCE is wrong here (neither sufficient nor necessary) codeinst = jl_type_infer(codeinst->def, jl_atomic_load_relaxed(&codeinst->max_world), SOURCE_MODE_FORCE_SOURCE); @@ -9935,6 +10216,16 @@ void jl_compile_workqueue( if (proto.specsig) { // expected specsig if (!preal_specsig) { + if (params.params->trim) { + auto it = params.compiled_functions.find(codeinst); //TODO: What to do about this + errs() << "Bailed out to invoke when compiling:"; + jl_(codeinst->def); + if (it != params.compiled_functions.end()) { + errs() << it->second.second.functionObject << "\n"; + errs() << it->second.second.specFunctionObject << "\n"; + } else + errs() << "codeinst not in compile_functions\n"; + } // emit specsig-to-(jl)invoke conversion StringRef invokeName; if (invoke != NULL) @@ -10124,6 +10415,22 @@ int jl_opaque_ptrs_set = 0; extern "C" void jl_init_llvm(void) { + jl_default_cgparams = { + /* track_allocations */ 1, + /* code_coverage */ 1, + /* prefer_specsig */ 0, +#ifdef _OS_WINDOWS_ + /* gnu_pubnames */ 0, +#else + /* gnu_pubnames */ 1, +#endif + /* debug_info_kind */ (int) DICompileUnit::DebugEmissionKind::FullDebug, + /* debug_info_level */ (int) jl_options.debug_level, + /* safepoint_on_entry */ 1, + /* gcstack_arg */ 1, + /* use_jlplt*/ 1, + /* trim */ 0, + /* lookup */ jl_rettype_inferred_addr }; jl_page_size = jl_getpagesize(); jl_default_debug_info_kind = (int) DICompileUnit::DebugEmissionKind::FullDebug; jl_default_cgparams.debug_info_level = (int) jl_options.debug_level; diff --git a/src/gf.c b/src/gf.c index e6f5b4ee007f7..321711c839aa8 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1360,8 +1360,7 @@ static inline jl_typemap_entry_t *lookup_leafcache(jl_genericmemory_t *leafcache } return NULL; } - -static jl_method_instance_t *cache_method( +jl_method_instance_t *cache_method( jl_methtable_t *mt, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, jl_tupletype_t *tt, // the original tupletype of the signature jl_method_t *definition, @@ -1707,7 +1706,7 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue jl_printf(s, ".\n"); jl_uv_flush(s); } - if (jl_generating_output()) { + if (jl_generating_output() && jl_options.incremental) { jl_printf(JL_STDERR, "ERROR: Method overwriting is not permitted during Module precompilation. Use `__precompile__(false)` to opt-out of precompilation.\n"); jl_throw(jl_precompilable_error); } @@ -2411,7 +2410,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t na // spvals is any matched static parameter values, m is the Method, // full is a boolean indicating if that method fully covers the input // -// lim is the max # of methods to return. if there are more, returns jl_false. +// lim is the max # of methods to return. if there are more, returns jl_nothing. // Negative values stand for no limit. // Unless lim == -1, remove matches that are unambiguously covered by earlier ones JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t *mt, int lim, int include_ambiguous, @@ -2431,7 +2430,7 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t * return ml_matches((jl_methtable_t*)mt, types, lim, include_ambiguous, 1, world, 1, min_valid, max_valid, ambig); } -jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT) +JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT) { // one unspecialized version of a function can be shared among all cached specializations if (!jl_is_method(def) || def->source == NULL) { @@ -2910,7 +2909,7 @@ jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_ } // return a MethodInstance for a compileable method_match -jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache) +JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache) { jl_method_t *m = match->method; jl_svec_t *env = match->sparams; @@ -3112,6 +3111,21 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) return 1; } +JL_DLLEXPORT int jl_add_entrypoint(jl_tupletype_t *types) +{ + size_t world = jl_atomic_load_acquire(&jl_world_counter); + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + jl_method_instance_t *mi = jl_get_compile_hint_specialization(types, world, &min_valid, &max_valid, 1); + if (mi == NULL) + return 0; + JL_GC_PROMISE_ROOTED(mi); + if (jl_generating_output() && jl_options.trim) { + arraylist_push(jl_entrypoint_mis, mi); + } + return 1; +} + // add type of `f` to front of argument tuple type jl_value_t *jl_argtype_with_function(jl_value_t *f, jl_value_t *types0) { diff --git a/src/init.c b/src/init.c index 86c0877b14289..413d4e8055e54 100644 --- a/src/init.c +++ b/src/init.c @@ -44,6 +44,7 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); // list of modules being deserialized with __init__ methods jl_array_t *jl_module_init_order; +arraylist_t *jl_entrypoint_mis; JL_DLLEXPORT size_t jl_page_size; @@ -721,6 +722,7 @@ static void restore_fp_env(void) static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_task_t *ct); JL_DLLEXPORT int jl_default_debug_info_kind; +JL_DLLEXPORT jl_cgparams_t jl_default_cgparams; static void init_global_mutexes(void) { JL_MUTEX_INIT(&jl_modules_mutex, "jl_modules_mutex"); @@ -841,8 +843,10 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ JL_TIMING(JULIA_INIT, JULIA_INIT); jl_resolve_sysimg_location(rel); // loads sysimg if available, and conditionally sets jl_options.cpu_target - if (rel == JL_IMAGE_IN_MEMORY) + if (rel == JL_IMAGE_IN_MEMORY) { jl_set_sysimg_so(jl_exe_handle); + jl_options.image_file = jl_options.julia_bin; + } else if (jl_options.image_file) jl_preload_sysimg_so(jl_options.image_file); if (jl_options.cpu_target == NULL) @@ -899,6 +903,11 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ JL_GC_POP(); } + if (jl_options.trim) { + jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); + arraylist_new(jl_entrypoint_mis, 0); + } + if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) jl_install_sigint_handler(); } diff --git a/src/jitlayers.h b/src/jitlayers.h index 107782e354d4a..93669c2351d88 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -29,7 +29,7 @@ #include "llvm-version.h" #include #include - +#include // As of LLVM 13, there are two runtime JIT linker implementations, the older // RuntimeDyld (used via orc::RTDyldObjectLinkingLayer) and the newer JITLink @@ -65,6 +65,7 @@ using namespace llvm; extern "C" jl_cgparams_t jl_default_cgparams; +extern arraylist_t new_invokes; DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeContext, LLVMOrcThreadSafeContextRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule, LLVMOrcThreadSafeModuleRef) @@ -211,7 +212,7 @@ struct jl_codegen_call_target_t { typedef SmallVector, 0> jl_workqueue_t; // TODO DenseMap? typedef std::map> jl_compiled_functions_t; - +typedef std::list> CallFrames; struct jl_codegen_params_t { orc::ThreadSafeContext tsctx; orc::ThreadSafeContext::Lock tsctx_lock; @@ -230,6 +231,7 @@ struct jl_codegen_params_t { std::map ditypes; std::map llvmtypes; DenseMap mergedConstants; + llvm::MapVector> enqueuers; // Map from symbol name (in a certain library) to its GV in sysimg and the // DL handle address in the current session. StringMap> libMapGV; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 7abf2b055bb8c..0c712ef37cb5b 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -344,6 +344,8 @@ XX(jl_new_typevar) \ XX(jl_next_from_addrinfo) \ XX(jl_normalize_to_compilable_sig) \ + XX(jl_method_match_to_mi) \ + XX(jl_get_unspecialized) \ XX(jl_no_exc_handler) \ XX(jl_object_id) \ XX(jl_object_id_) \ @@ -522,6 +524,7 @@ YY(jl_dump_native) \ YY(jl_get_llvm_gvs) \ YY(jl_get_llvm_external_fns) \ + YY(jl_get_llvm_mis) \ YY(jl_dump_function_asm) \ YY(jl_LLVMCreateDisasm) \ YY(jl_LLVMDisasmInstruction) \ diff --git a/src/jloptions.c b/src/jloptions.c index f63f4de020e26..530d5e2577a9a 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -101,6 +101,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // permalloc_pkgimg 0, // heap-size-hint 0, // trace_compile_timing + 0, // trim }; jl_options_initialized = 1; } @@ -251,7 +252,7 @@ static const char opts_hidden[] = " --strip-ir Remove IR (intermediate representation) of compiled\n" " functions\n\n" - // compiler debugging (see the devdocs for tips on using these options) + // compiler debugging and experimental (see the devdocs for tips on using these options) " --output-unopt-bc Generate unoptimized LLVM bitcode (.bc)\n" " --output-bc Generate LLVM bitcode (.bc)\n" " --output-asm Generate an assembly file (.s)\n" @@ -265,6 +266,13 @@ static const char opts_hidden[] = " compile in ms\n" " --image-codegen Force generate code in imaging mode\n" " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n" + " --trim={no*|safe|unsafe|unsafe-warn}\n" + " Build a sysimage including only code provably reachable\n" + " from methods marked by calling `entrypoint`. In unsafe\n" + " mode, the resulting binary might be missing needed code\n" + " and can throw errors. With unsafe-warn warnings will be\n" + " printed for dynamic call sites that might lead to such\n" + " errors. In safe mode compile-time errors are given instead.\n" ; JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) @@ -311,7 +319,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_ir, opt_heap_size_hint, opt_gc_threads, - opt_permalloc_pkgimg + opt_permalloc_pkgimg, + opt_trim, }; static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:m:"; static const struct option longopts[] = { @@ -375,6 +384,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -934,6 +944,18 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --permalloc-pkgimg={yes|no} (%s)", optarg); break; + case opt_trim: + if (optarg == NULL || !strcmp(optarg,"safe")) + jl_options.trim = JL_TRIM_SAFE; + else if (!strcmp(optarg,"no")) + jl_options.trim = JL_TRIM_NO; + else if (!strcmp(optarg,"unsafe")) + jl_options.trim = JL_TRIM_UNSAFE; + else if (!strcmp(optarg,"unsafe-warn")) + jl_options.trim = JL_TRIM_UNSAFE_WARN; + else + jl_errorf("julia: invalid argument to --trim={safe|no|unsafe|unsafe-warn} (%s)", optarg); + break; default: jl_errorf("julia: unhandled option -- %c\n" "This is a bug, please report it.", c); diff --git a/src/jloptions.h b/src/jloptions.h index aac2a64a373a8..3d7deedb59e15 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -62,6 +62,7 @@ typedef struct { int8_t permalloc_pkgimg; uint64_t heap_size_hint; int8_t trace_compile_timing; + int8_t trim; } jl_options_t; #endif diff --git a/src/julia.expmap.in b/src/julia.expmap.in index e5f9ee890205f..29366f6296a85 100644 --- a/src/julia.expmap.in +++ b/src/julia.expmap.in @@ -5,8 +5,8 @@ asprintf; bitvector_*; ios_*; - arraylist_grow; - small_arraylist_grow; + arraylist_*; + small_arraylist_*; jl_*; ijl_*; _jl_mutex_*; diff --git a/src/julia.h b/src/julia.h index abb8a57ff13b0..73b96cf0183d1 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2579,6 +2579,11 @@ JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; #define JL_OPTIONS_USE_PKGIMAGES_YES 1 #define JL_OPTIONS_USE_PKGIMAGES_NO 0 +#define JL_TRIM_NO 0 +#define JL_TRIM_SAFE 1 +#define JL_TRIM_UNSAFE 2 +#define JL_TRIM_UNSAFE_WARN 3 + // Version information #include // Generated file @@ -2626,10 +2631,12 @@ typedef struct { int gcstack_arg; // Pass the ptls value as an argument with swiftself int use_jlplt; // Whether to use the Julia PLT mechanism or emit symbols directly + int trim; // can we emit dynamic dispatches? // Cache access. Default: jl_rettype_inferred_native. jl_codeinstance_lookup_t lookup; } jl_cgparams_t; extern JL_DLLEXPORT int jl_default_debug_info_kind; +extern JL_DLLEXPORT jl_cgparams_t jl_default_cgparams; typedef struct { int emit_metadata; diff --git a/src/julia_internal.h b/src/julia_internal.h index f00667d016796..9a61c3d18356f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -664,8 +664,9 @@ JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_value_t *rettype, size_t min_world, size_t max_world, jl_debuginfo_t *edges); -jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT); JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner); JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( @@ -683,6 +684,7 @@ JL_DLLEXPORT const char *jl_debuginfo_name(jl_value_t *func) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_compile_method_instance(jl_method_instance_t *mi, jl_tupletype_t *types, size_t world); JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types); +JL_DLLEXPORT int jl_add_entrypoint(jl_tupletype_t *types); jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *lam JL_PROPAGATES_ROOT, size_t world); jl_value_t *jl_code_or_ci_for_interpreter(jl_method_instance_t *lam JL_PROPAGATES_ROOT, size_t world); int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile); @@ -853,6 +855,7 @@ extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED; +extern arraylist_t *jl_entrypoint_mis; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val, int insert) JL_GLOBALLY_ROOTED; @@ -1902,7 +1905,7 @@ JL_DLLIMPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncod int32_t *func_idx, int32_t *specfunc_idx); JL_DLLIMPORT void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n); - +JL_DLLIMPORT void jl_get_llvm_mis(void *native_code, arraylist_t* MIs); JL_DLLIMPORT void jl_init_codegen(void); JL_DLLIMPORT void jl_teardown_codegen(void) JL_NOTSAFEPOINT; JL_DLLIMPORT int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline) JL_NOTSAFEPOINT; diff --git a/src/module.c b/src/module.c index 96d94049cff13..a6c05d279f5b0 100644 --- a/src/module.c +++ b/src/module.c @@ -856,7 +856,7 @@ JL_DLLEXPORT int jl_binding_resolved_p(jl_module_t *m, jl_sym_t *var) return kind == BINDING_KIND_DECLARED || !jl_bkind_is_some_guard(kind); } -static uint_t bindingkey_hash(size_t idx, jl_value_t *data) +uint_t bindingkey_hash(size_t idx, jl_value_t *data) { jl_binding_t *b = (jl_binding_t*)jl_svecref(data, idx); // This must always happen inside the lock jl_sym_t *var = b->globalref->name; diff --git a/src/precompile.c b/src/precompile.c index c40e867ea699e..5088d45a5ad74 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -116,14 +116,16 @@ JL_DLLEXPORT void jl_write_compiler_output(void) if (f) { jl_array_ptr_1d_push(jl_module_init_order, m); int setting = jl_get_module_compile((jl_module_t*)m); - if (setting != JL_OPTIONS_COMPILE_OFF && - setting != JL_OPTIONS_COMPILE_MIN) { + if ((setting != JL_OPTIONS_COMPILE_OFF && (jl_options.trim || + (setting != JL_OPTIONS_COMPILE_MIN)))) { // TODO: this would be better handled if moved entirely to jl_precompile // since it's a slightly duplication of effort jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = jl_apply_tuple_type_v(&tt, 1); jl_compile_hint((jl_tupletype_t*)tt); + if (jl_options.trim) + jl_add_entrypoint((jl_tupletype_t*)tt); JL_GC_POP(); } } @@ -188,6 +190,10 @@ JL_DLLEXPORT void jl_write_compiler_output(void) jl_printf(JL_STDERR, "\n ** incremental compilation may be broken for this module **\n\n"); } } + if (jl_options.trim) { + exit(0); // Some finalizers need to run and we've blown up the bindings table + // TODO: Is this still needed + } JL_GC_POP(); jl_gc_enable_finalizers(ct, 1); } diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 5a4f599d1f0eb..a78d1e66dbb51 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -321,3 +321,83 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met JL_GC_POP(); return native_code; } + +static int enq_ccallable_entrypoints_(jl_typemap_entry_t *def, void *closure) +{ + jl_method_t *m = def->func.method; + if (m->external_mt) + return 1; + if (m->ccallable) + jl_add_entrypoint((jl_tupletype_t*)jl_svecref(m->ccallable, 1)); + return 1; +} + +static int enq_ccallable_entrypoints(jl_methtable_t *mt, void *env) +{ + return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), enq_ccallable_entrypoints_, env); +} + +JL_DLLEXPORT void jl_add_ccallable_entrypoints(void) +{ + jl_foreach_reachable_mtable(enq_ccallable_entrypoints, NULL); +} + +static void *jl_precompile_trimmed(size_t world) +{ + // array of MethodInstances and ccallable aliases to include in the output + jl_array_t *m = jl_alloc_vec_any(0); + jl_value_t *ccallable = NULL; + JL_GC_PUSH2(&m, &ccallable); + jl_method_instance_t *mi; + while (1) + { + mi = (jl_method_instance_t*)arraylist_pop(jl_entrypoint_mis); + if (mi == NULL) + break; + assert(jl_is_method_instance(mi)); + + jl_array_ptr_1d_push(m, (jl_value_t*)mi); + ccallable = (jl_value_t *)mi->def.method->ccallable; + if (ccallable) + jl_array_ptr_1d_push(m, ccallable); + } + + jl_cgparams_t params = jl_default_cgparams; + params.trim = jl_options.trim; + void *native_code = jl_create_native(m, NULL, ¶ms, 0, /* imaging */ 1, 0, + world); + JL_GC_POP(); + return native_code; +} + +static void jl_rebuild_methtables(arraylist_t* MIs, htable_t* mtables) +{ + size_t i; + for (i = 0; i < MIs->len; i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)MIs->items[i]; + jl_method_t *m = mi->def.method; + jl_methtable_t *old_mt = jl_method_get_table(m); + if ((jl_value_t *)old_mt == jl_nothing) + continue; + jl_sym_t *name = old_mt->name; + if (!ptrhash_has(mtables, old_mt)) + ptrhash_put(mtables, old_mt, jl_new_method_table(name, m->module)); + jl_methtable_t *mt = (jl_methtable_t*)ptrhash_get(mtables, old_mt); + size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t * lookup = jl_methtable_lookup(mt, m->sig, world); + // Check if the method is already in the new table, if not then insert it there + if (lookup == jl_nothing || (jl_method_t*)lookup != m) { + //TODO: should this be a function like unsafe_insert_method? + size_t min_world = jl_atomic_load_relaxed(&m->primary_world); + size_t max_world = jl_atomic_load_relaxed(&m->deleted_world); + jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); + jl_atomic_store_relaxed(&m->deleted_world, 1); + jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); + jl_atomic_store_relaxed(&m->primary_world, min_world); + jl_atomic_store_relaxed(&m->deleted_world, max_world); + jl_atomic_store_relaxed(&newentry->min_world, min_world); + jl_atomic_store_relaxed(&newentry->max_world, max_world); + } + } + +} diff --git a/src/staticdata.c b/src/staticdata.c index 363aa46b62221..f54cc9692eaea 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -363,6 +363,9 @@ static void *to_seroder_entry(size_t idx) return (void*)((char*)HT_NOTFOUND + 1 + idx); } +static htable_t new_methtables; +static size_t precompilation_world; + static int ptr_cmp(const void *l, const void *r) { uintptr_t left = *(const uintptr_t*)l; @@ -770,22 +773,41 @@ static uintptr_t jl_fptr_id(void *fptr) #define jl_queue_for_serialization(s, v) jl_queue_for_serialization_((s), (jl_value_t*)(v), 1, 0) static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED; - static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m) JL_GC_DISABLED { jl_queue_for_serialization(s, m->name); jl_queue_for_serialization(s, m->parent); - jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); + if (jl_options.trim) { + jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); + } else { + jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); + } jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset)); - if (jl_options.strip_metadata) { + if (jl_options.strip_metadata || jl_options.trim) { jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); for (size_t i = 0; i < jl_svec_len(table); i++) { jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); if ((void*)b == jl_nothing) break; - jl_sym_t *name = b->globalref->name; - if (name == jl_docmeta_sym && jl_get_binding_value(b)) - record_field_change((jl_value_t**)&b->value, jl_nothing); + if (jl_options.strip_metadata) { + jl_sym_t *name = b->globalref->name; + if (name == jl_docmeta_sym && jl_get_binding_value(b)) + record_field_change((jl_value_t**)&b->value, jl_nothing); + } + if (jl_options.trim) { + jl_value_t *val = jl_get_binding_value(b); + // keep binding objects that are defined and ... + if (val && + // ... point to modules ... + (jl_is_module(val) || + // ... or point to __init__ methods ... + !strcmp(jl_symbol_name(b->globalref->name), "__init__") || + // ... or point to Base functions accessed by the runtime + (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || + !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { + jl_queue_for_serialization(s, b); + } + } } } @@ -944,6 +966,23 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ jl_queue_for_serialization_(s, get_replaceable_field((jl_value_t**)&bpart->next, 0), 1, immediate); } else if (layout->nfields > 0) { + if (jl_options.trim) { + if (jl_is_method(v)) { + jl_method_t *m = (jl_method_t *)v; + if (jl_is_svec(jl_atomic_load_relaxed(&m->specializations))) + jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->specializations), 0, 1); + } + else if (jl_typetagis(v, jl_typename_type)) { + jl_typename_t *tn = (jl_typename_t*)v; + if (tn->mt != NULL && !tn->mt->frozen) { + jl_methtable_t * new_methtable = (jl_methtable_t *)ptrhash_get(&new_methtables, tn->mt); + if (new_methtable != HT_NOTFOUND) + record_field_change((jl_value_t **)&tn->mt, (jl_value_t*)new_methtable); + else + record_field_change((jl_value_t **)&tn->mt, NULL); + } + } + } char *data = (char*)jl_data_ptr(v); size_t i, np = layout->npointers; for (i = 0; i < np; i++) { @@ -989,6 +1028,7 @@ done_fields: ; } } + static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED { if (!jl_needs_serialization(s, v)) @@ -2407,6 +2447,53 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } +uint_t bindingkey_hash(size_t idx, jl_value_t *data); + +static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED +{ + jl_svec_t * bindings = jl_atomic_load_relaxed(&m->bindings); + size_t l = jl_svec_len(bindings), i; + arraylist_t bindings_list; + arraylist_new(&bindings_list, 0); + if (l == 0) + return; + for (i = 0; i < l; i++) { + jl_value_t *ti = jl_svecref(bindings, i); + if (ti == jl_nothing) + continue; + jl_binding_t *ref = ((jl_binding_t*)ti); + if (!((ptrhash_get(&serialization_order, ref) == HT_NOTFOUND) && + (ptrhash_get(&serialization_order, ref->globalref) == HT_NOTFOUND))) { + jl_svecset(bindings, i, jl_nothing); + arraylist_push(&bindings_list, ref); + } + } + jl_genericmemory_t* bindingkeyset = jl_atomic_load_relaxed(&m->bindingkeyset); + _Atomic(jl_genericmemory_t*)bindingkeyset2; + jl_atomic_store_relaxed(&bindingkeyset2,(jl_genericmemory_t*)jl_an_empty_memory_any); + jl_svec_t *bindings2 = jl_alloc_svec_uninit(bindings_list.len); + for (i = 0; i < bindings_list.len; i++) { + jl_binding_t *ref = (jl_binding_t*)bindings_list.items[i]; + jl_svecset(bindings2, i, ref); + jl_smallintset_insert(&bindingkeyset2, (jl_value_t*)m, bindingkey_hash, i, (jl_value_t*)bindings2); + } + void *idx = ptrhash_get(&serialization_order, bindings); + assert(idx != HT_NOTFOUND && idx != (void*)(uintptr_t)-1); + assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == bindings); + ptrhash_put(&serialization_order, bindings2, idx); + serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = bindings2; + + idx = ptrhash_get(&serialization_order, bindingkeyset); + assert(idx != HT_NOTFOUND && idx != (void*)(uintptr_t)-1); + assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == bindingkeyset); + ptrhash_put(&serialization_order, jl_atomic_load_relaxed(&bindingkeyset2), idx); + serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = jl_atomic_load_relaxed(&bindingkeyset2); + jl_atomic_store_relaxed(&m->bindings, bindings2); + jl_atomic_store_relaxed(&m->bindingkeyset, jl_atomic_load_relaxed(&bindingkeyset2)); + jl_gc_wb(m, bindings2); + jl_gc_wb(m, jl_atomic_load_relaxed(&bindingkeyset2)); +} + static void strip_slotnames(jl_array_t *slotnames) { // replace slot names with `?`, except unused_sym since the compiler looks at it @@ -2473,7 +2560,7 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) if (m->source) { int stripped_ir = 0; if (jl_options.strip_ir) { - int should_strip_ir = 0; + int should_strip_ir = jl_options.trim; if (!should_strip_ir) { if (jl_atomic_load_relaxed(&m->unspecialized)) { jl_code_instance_t *unspec = jl_atomic_load_relaxed(&jl_atomic_load_relaxed(&m->unspecialized)->cache); @@ -2675,8 +2762,46 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // strip metadata and IR when requested if (jl_options.strip_metadata || jl_options.strip_ir) jl_strip_all_codeinfos(); + // collect needed methods and replace method tables that are in the tags array + htable_new(&new_methtables, 0); + arraylist_t MIs; + arraylist_new(&MIs, 0); + arraylist_t gvars; + arraylist_new(&gvars, 0); + arraylist_t external_fns; + arraylist_new(&external_fns, 0); int en = jl_gc_enable(0); + if (native_functions) { + jl_get_llvm_gvs(native_functions, &gvars); + jl_get_llvm_external_fns(native_functions, &external_fns); + if (jl_options.trim) + jl_get_llvm_mis(native_functions, &MIs); + } + if (jl_options.trim) { + jl_rebuild_methtables(&MIs, &new_methtables); + jl_methtable_t *mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_type_type_mt); + JL_GC_PROMISE_ROOTED(mt); + if (mt != HT_NOTFOUND) + jl_type_type_mt = mt; + else + jl_type_type_mt = jl_new_method_table(jl_type_type_mt->name, jl_type_type_mt->module); + + mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_kwcall_mt); + JL_GC_PROMISE_ROOTED(mt); + if (mt != HT_NOTFOUND) + jl_kwcall_mt = mt; + else + jl_kwcall_mt = jl_new_method_table(jl_kwcall_mt->name, jl_kwcall_mt->module); + + mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_nonfunction_mt); + JL_GC_PROMISE_ROOTED(mt); + if (mt != HT_NOTFOUND) + jl_nonfunction_mt = mt; + else + jl_nonfunction_mt = jl_new_method_table(jl_nonfunction_mt->name, jl_nonfunction_mt->module); + } + nsym_tag = 0; htable_new(&symbol_table, 0); htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs)); @@ -2722,14 +2847,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; - arraylist_t gvars; - arraylist_t external_fns; - arraylist_new(&gvars, 0); - arraylist_new(&external_fns, 0); - if (native_functions) { - jl_get_llvm_gvs(native_functions, &gvars); - jl_get_llvm_external_fns(native_functions, &external_fns); - } if (worklist == NULL) { // empty!(Core.ARGS) @@ -2788,6 +2905,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // step 1.2: ensure all gvars are part of the sysimage too record_gvars(&s, &gvars); record_external_fns(&s, &external_fns); + if (jl_options.trim) + record_gvars(&s, &MIs); jl_serialize_reachable(&s); // step 1.3: prune (garbage collect) special weak references from the jl_global_roots_list if (worklist == NULL) { @@ -2808,8 +2927,30 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // step 1.4: prune (garbage collect) some special weak references from // built-in type caches too for (i = 0; i < serialization_queue.len; i++) { - jl_typename_t *tn = (jl_typename_t*)serialization_queue.items[i]; - if (jl_is_typename(tn)) { + jl_value_t *v = (jl_value_t*)serialization_queue.items[i]; + if (jl_options.trim) { + if (jl_is_method(v)){ + jl_method_t *m = (jl_method_t*)v; + jl_value_t *specializations_ = jl_atomic_load_relaxed(&m->specializations); + if (!jl_is_svec(specializations_)) + continue; + + jl_svec_t *specializations = (jl_svec_t *)specializations_; + size_t l = jl_svec_len(specializations), i; + for (i = 0; i < l; i++) { + jl_value_t *mi = jl_svecref(specializations, i); + if (mi == jl_nothing) + continue; + if (ptrhash_get(&serialization_order, mi) == HT_NOTFOUND) + jl_svecset(specializations, i, jl_nothing); + } + } else if (jl_is_module(v)) { + jl_prune_module_bindings((jl_module_t*)v); + } + } + // Not else + if (jl_is_typename(v)) { + jl_typename_t *tn = (jl_typename_t*)v; jl_atomic_store_relaxed(&tn->cache, jl_prune_type_cache_hash(jl_atomic_load_relaxed(&tn->cache))); jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); @@ -2918,7 +3059,9 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_write_value(&s, global_roots_keyset); jl_write_value(&s, s.ptls->root_task->tls); write_uint32(f, jl_get_gs_ctr()); - write_uint(f, jl_atomic_load_acquire(&jl_world_counter)); + size_t world = jl_atomic_load_acquire(&jl_world_counter); + // assert(world == precompilation_world); // This triggers on a normal build of julia + write_uint(f, world); write_uint(f, jl_typeinf_world); } else { @@ -2971,6 +3114,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, htable_free(&nullptrs); htable_free(&symbol_table); htable_free(&fptr_to_id); + htable_free(&new_methtables); nsym_tag = 0; jl_gc_enable(en); @@ -3000,6 +3144,10 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos) { + if (jl_options.strip_ir || jl_options.trim) { + // make sure this is precompiled for jl_foreach_reachable_mtable + jl_get_loaded_modules(); + } jl_gc_collect(JL_GC_FULL); jl_gc_collect(JL_GC_INCREMENTAL); // sweep finalizers JL_TIMING(SYSIMG_DUMP, SYSIMG_DUMP); @@ -3049,7 +3197,11 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } } else if (_native_data != NULL) { - *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); + precompilation_world = jl_atomic_load_acquire(&jl_world_counter); + if (jl_options.trim) + *_native_data = jl_precompile_trimmed(precompilation_world); + else + *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); } // Make sure we don't run any Julia code concurrently after this point diff --git a/src/support/arraylist.h b/src/support/arraylist.h index 6ad2f0e2f28c9..a83bd2808756c 100644 --- a/src/support/arraylist.h +++ b/src/support/arraylist.h @@ -20,11 +20,11 @@ typedef struct { void *_space[AL_N_INLINE]; } arraylist_t; -arraylist_t *arraylist_new(arraylist_t *a, size_t size) JL_NOTSAFEPOINT; -void arraylist_free(arraylist_t *a) JL_NOTSAFEPOINT; +JL_DLLEXPORT arraylist_t *arraylist_new(arraylist_t *a, size_t size) JL_NOTSAFEPOINT; +JL_DLLEXPORT void arraylist_free(arraylist_t *a) JL_NOTSAFEPOINT; -void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT; -void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT; +JL_DLLEXPORT void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT; +JL_DLLEXPORT void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT; JL_DLLEXPORT void arraylist_grow(arraylist_t *a, size_t n) JL_NOTSAFEPOINT; typedef struct { @@ -34,11 +34,12 @@ typedef struct { void *_space[SMALL_AL_N_INLINE]; } small_arraylist_t; -small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size) JL_NOTSAFEPOINT; -void small_arraylist_free(small_arraylist_t *a) JL_NOTSAFEPOINT; -void small_arraylist_push(small_arraylist_t *a, void *elt) JL_NOTSAFEPOINT; -void *small_arraylist_pop(small_arraylist_t *a) JL_NOTSAFEPOINT; +JL_DLLEXPORT small_arraylist_t *small_arraylist_new(small_arraylist_t *a, uint32_t size) JL_NOTSAFEPOINT; +JL_DLLEXPORT void small_arraylist_free(small_arraylist_t *a) JL_NOTSAFEPOINT; + +JL_DLLEXPORT void small_arraylist_push(small_arraylist_t *a, void *elt) JL_NOTSAFEPOINT; +JL_DLLEXPORT void *small_arraylist_pop(small_arraylist_t *a) JL_NOTSAFEPOINT; JL_DLLEXPORT void small_arraylist_grow(small_arraylist_t *a, uint32_t n) JL_NOTSAFEPOINT; #ifdef __cplusplus diff --git a/stdlib/LinearAlgebra/src/blas.jl b/stdlib/LinearAlgebra/src/blas.jl index 413b7866c5444..3c15630091162 100644 --- a/stdlib/LinearAlgebra/src/blas.jl +++ b/stdlib/LinearAlgebra/src/blas.jl @@ -159,7 +159,7 @@ function check() interface = USE_BLAS64 ? :ilp64 : :lp64 if !any(lib.interface == interface for lib in config.loaded_libs) interfacestr = uppercase(string(interface)) - @error("No loaded BLAS libraries were built with $interfacestr support.") + println(Core.stderr, "No loaded BLAS libraries were built with $interfacestr support.") exit(1) end end diff --git a/stdlib/LinearAlgebra/src/lbt.jl b/stdlib/LinearAlgebra/src/lbt.jl index 606ddedbe1343..81d10f930c8c5 100644 --- a/stdlib/LinearAlgebra/src/lbt.jl +++ b/stdlib/LinearAlgebra/src/lbt.jl @@ -17,7 +17,7 @@ end macro get_warn(map, key) return quote if !haskey($(esc(map)), $(esc(key))) - @warn(string("[LBT] Unknown key into ", $(string(map)), ": ", $(esc(key)), ", defaulting to :unknown")) + println(Core.stderr, string("Warning: [LBT] Unknown key into ", $(string(map)), ": ", $(esc(key)), ", defaulting to :unknown")) # All the unknown values share a common value: `-1` $(esc(map))[$(esc(LBT_INTERFACE_UNKNOWN))] else @@ -132,7 +132,7 @@ struct LBTConfig if str_ptr != C_NULL push!(exported_symbols, unsafe_string(str_ptr)) else - @error("NULL string in lbt_config.exported_symbols[$(sym_idx)]") + println(Core.stderr, "Error: NULL string in lbt_config.exported_symbols[$(sym_idx)]") end end diff --git a/test/Makefile b/test/Makefile index 1b9cb377c943d..6ebdd3c764fd5 100644 --- a/test/Makefile +++ b/test/Makefile @@ -24,6 +24,8 @@ EMBEDDING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/embedding" "CC=$(CC GCEXT_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/gcext" "CC=$(CC)" +TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(JULIAHOME)/usr/bin" "CC=$(CC)" + default: $(TESTS): @@ -66,6 +68,9 @@ embedding: gcext: @$(MAKE) -C $(SRCDIR)/$@ check $(GCEXT_ARGS) +trimming: + @$(MAKE) -C $(SRCDIR)/$@ check $(TRIMMING_ARGS) + clangsa: @$(MAKE) -C $(SRCDIR)/$@ @@ -73,5 +78,6 @@ clean: @$(MAKE) -C embedding $@ $(EMBEDDING_ARGS) @$(MAKE) -C gcext $@ $(GCEXT_ARGS) @$(MAKE) -C llvmpasses $@ + @$(MAKE) -C trimming $@ $(TRIMMING_ARGS) -.PHONY: $(TESTS) $(addprefix revise-, $(TESTS)) relocatedepot revise-relocatedepot embedding gcext clangsa clean +.PHONY: $(TESTS) $(addprefix revise-, $(TESTS)) relocatedepot revise-relocatedepot embedding gcext trimming clangsa clean diff --git a/test/trimming/Makefile b/test/trimming/Makefile new file mode 100644 index 0000000000000..c6e105d637013 --- /dev/null +++ b/test/trimming/Makefile @@ -0,0 +1,55 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# This Makefile template requires the following variables to be set +# in the environment or on the command-line: +# JULIA: path to julia[.exe] executable +# BIN: binary build directory + +ifndef JULIA + $(error "Please pass JULIA=[path of target julia binary], or set as environment variable!") +endif +ifndef BIN + $(error "Please pass BIN=[path of build directory], or set as environment variable!") +endif + +#============================================================================= +# location of test source +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/../..) +BUILDSCRIPT := $(BIN)/../share/julia/juliac-buildscript.jl +include $(JULIAHOME)/Make.inc + +# get the executable suffix, if any +EXE := $(suffix $(abspath $(JULIA))) + +# get compiler and linker flags. (see: `contrib/julia-config.jl`) +JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- +CPPFLAGS_ADD := +CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) +LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal + +#============================================================================= + +release: hello$(EXE) + +hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe true + +init.o: $(SRCDIR)/init.c + $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) + +hello$(EXE): hello.o init.o + $(CC) -o $@ $(WHOLE_ARCHIVE) hello.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) + +check: hello$(EXE) + $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming + +clean: + -rm -f hello$(EXE) init.o hello.o + +.PHONY: release clean check + +# Makefile debugging trick: +# call print-VARIABLE to see the runtime value of any variable +print-%: + @echo '$*=$($*)' diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl new file mode 100644 index 0000000000000..307bf820f325b --- /dev/null +++ b/test/trimming/hello.jl @@ -0,0 +1,6 @@ +module MyApp +Base.@ccallable function main()::Cint + println(Core.stdout, "Hello, world!") + return 0 +end +end diff --git a/test/trimming/init.c b/test/trimming/init.c new file mode 100644 index 0000000000000..ea1b02f8e5c8f --- /dev/null +++ b/test/trimming/init.c @@ -0,0 +1,9 @@ +#include + +__attribute__((constructor)) void static_init(void) +{ + if (jl_is_initialized()) + return; + julia_init(JL_IMAGE_IN_MEMORY); + jl_exception_clear(); +} diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl new file mode 100644 index 0000000000000..dfacae7f8e531 --- /dev/null +++ b/test/trimming/trimming.jl @@ -0,0 +1,7 @@ +using Test + +exe_path = joinpath(@__DIR__, "hello"*splitext(Base.julia_exename())[2]) + +@test readchomp(`$exe_path`) == "Hello, world!" + +@test filesize(exe_path) < filesize(unsafe_string(Base.JLOptions().image_file))/10