From 973aefe3334b5bb2416e7e0f248318becaac8328 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 6 Apr 2018 11:38:58 -0400 Subject: [PATCH] codegen: restructure, remove recursion also provides support for using a different code_native format, as a fallback, later we'll want to make this more configurable there are now several primary interfaces to native code: - codegen: mostly internal, support for translating IR to LLVM - jitlayers: manages runtime codegen results and executable memory - aotcompile: support for managing external code output - disasm: pretty-printer for code objects - debuginfo: tracking for unwind info also removes the global type caches and move all codegen pass handling to aotcompile.cpp --- base/compiler/typeinfer.jl | 6 +- base/options.jl | 1 - base/reflection.jl | 10 +- contrib/generate_precompile.jl | 2 +- doc/src/devdocs/llvm.md | 2 +- src/Makefile | 3 +- src/abi_ppc64le.cpp | 5 +- src/anticodegen.c | 3 +- src/aotcompile.cpp | 908 ++++++++++++ src/ccall.cpp | 106 +- src/cgutils.cpp | 369 ++--- src/codegen.cpp | 1602 ++++++++-------------- src/datatype.c | 2 - src/debuginfo.cpp | 18 +- src/disasm.cpp | 129 +- src/dump.c | 2 - src/gf.c | 89 +- src/intrinsics.cpp | 26 +- src/jitlayers.cpp | 1088 +++++---------- src/jitlayers.h | 100 +- src/jloptions.c | 6 - src/jltypes.c | 41 +- src/julia.h | 18 +- src/julia_internal.h | 32 +- src/precompile.c | 63 +- src/staticdata.c | 95 +- stdlib/InteractiveUtils/src/codeview.jl | 52 +- stdlib/InteractiveUtils/test/runtests.jl | 72 +- test/llvmcall.jl | 10 +- test/llvmcall2.jl | 8 + test/llvmpasses/loopinfo.jl | 34 +- test/reflection.jl | 21 +- test/staged.jl | 21 +- 33 files changed, 2606 insertions(+), 2338 deletions(-) create mode 100644 src/aotcompile.cpp diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 74f8695e111ec..32fce2da3ce1c 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -84,10 +84,6 @@ end function cache_result(result::InferenceResult, min_valid::UInt, max_valid::UInt) def = result.linfo.def toplevel = !isa(result.linfo.def, Method) - if toplevel - min_valid = UInt(0) - max_valid = UInt(0) - end # check if the existing linfo metadata is also sufficient to describe the current inference result # to decide if it is worth caching this @@ -546,7 +542,7 @@ function typeinf_ext(mi::MethodInstance, params::Params) tree.pure = true tree.inlineable = true tree.parent = mi - tree.rettype = typeof(code.rettype_const) + tree.rettype = Core.Typeof(code.rettype_const) tree.min_world = code.min_world tree.max_world = code.max_world return tree diff --git a/base/options.jl b/base/options.jl index 9716d77a9dc57..65f50fbd62814 100644 --- a/base/options.jl +++ b/base/options.jl @@ -36,7 +36,6 @@ struct JLOptions bindto::Ptr{UInt8} outputbc::Ptr{UInt8} outputunoptbc::Ptr{UInt8} - outputjitbc::Ptr{UInt8} outputo::Ptr{UInt8} outputji::Ptr{UInt8} output_code_coverage::Ptr{UInt8} diff --git a/base/reflection.jl b/base/reflection.jl index 813c1cdd08c4d..825e130cda85b 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -965,8 +965,6 @@ default_debug_info_kind() = unsafe_load(cglobal(:jl_default_debug_info_kind, Cin # this type mirrors jl_cgparams_t (documented in julia.h) struct CodegenParams - cached::Cint - track_allocations::Cint code_coverage::Cint static_alloc::Cint @@ -980,18 +978,18 @@ struct CodegenParams emit_function::Any emitted_function::Any - CodegenParams(;cached::Bool=true, - track_allocations::Bool=true, code_coverage::Bool=true, + function CodegenParams(; track_allocations::Bool=true, code_coverage::Bool=true, static_alloc::Bool=true, prefer_specsig::Bool=false, gnu_pubnames=true, debug_info_kind::Cint = default_debug_info_kind(), module_setup=nothing, module_activation=nothing, raise_exception=nothing, - emit_function=nothing, emitted_function=nothing) = - new(Cint(cached), + emit_function=nothing, emitted_function=nothing) + return new( Cint(track_allocations), Cint(code_coverage), Cint(static_alloc), Cint(prefer_specsig), Cint(gnu_pubnames), debug_info_kind, module_setup, module_activation, raise_exception, emit_function, emitted_function) + end end const SLOT_USED = 0x8 diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index f1e124a4f0e2b..b9762d79dd26a 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -175,7 +175,7 @@ function generate_precompile_statements() if have_repl # Seems like a reasonable number right now, adjust as needed # comment out if debugging script - @assert n_succeeded > 3500 + @assert n_succeeded > 1500 end print(" $(length(statements)) generated in ") diff --git a/doc/src/devdocs/llvm.md b/doc/src/devdocs/llvm.md index d89c8bfaf3076..c51ca8fd63191 100644 --- a/doc/src/devdocs/llvm.md +++ b/doc/src/devdocs/llvm.md @@ -97,7 +97,7 @@ using: fun, T = +, Tuple{Int,Int} # Substitute your function of interest here optimize = false open("plus.ll", "w") do file - println(file, InteractiveUtils._dump_function(fun, T, false, false, false, true, :att, optimize)) + println(file, InteractiveUtils._dump_function(fun, T, false, false, false, true, :att, optimize, :default)) end ``` These files can be processed the same way as the unoptimized sysimg IR shown diff --git a/src/Makefile b/src/Makefile index e9af00060b93c..03fbf1ffa4baa 100644 --- a/src/Makefile +++ b/src/Makefile @@ -52,7 +52,7 @@ LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) SRCS += codegen llvm-ptls -RUNTIME_SRCS += jitlayers debuginfo disasm llvm-simdloop llvm-muladd \ +RUNTIME_SRCS += jitlayers aotcompile debuginfo disasm llvm-simdloop llvm-muladd \ llvm-final-gc-lowering llvm-pass-helpers llvm-late-gc-lowering \ llvm-lower-handlers llvm-gc-invariant-verifier llvm-propagate-addrspaces \ llvm-multiversioning llvm-alloc-opt cgmemmgr llvm-api @@ -219,6 +219,7 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase # additional dependency links $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/builtin_proto.h $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\ diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index d13cd35853ade..3c13d8b47aa4d 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -133,10 +133,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override else { jl_datatype_t *vecty = (jl_datatype_t*)jl_field_type(ty0, 0); assert(jl_is_datatype(vecty) && vecty->name == jl_vecelement_typename); - jl_value_t *elemty = jl_tparam0(vecty); - assert(jl_is_primitivetype(elemty)); - - Type *ety = julia_type_to_llvm(elemty); + Type *ety = bitstype_to_llvm(jl_tparam0(vecty)); Type *vty = VectorType::get(ety, jl_datatype_nfields(ty0)); return ArrayType::get(vty, hfa); } diff --git a/src/anticodegen.c b/src/anticodegen.c index 088ce0d1f0d82..e93ba05c47a0b 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -17,8 +17,9 @@ void jl_write_coverage_data(void) UNAVAILABLE JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) UNAVAILABLE -JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc, const char* asm_variant, const char *debuginfo) UNAVAILABLE +JL_DLLEXPORT jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world, int raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo) UNAVAILABLE JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module, const char *debuginfo) UNAVAILABLE +JL_DLLEXPORT void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params) UNAVAILABLE JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, void *GetOpInfo, void *SymbolLookUp) UNAVAILABLE JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *Bytes, uint64_t BytesSize, uint64_t PC, char *OutString, size_t OutStringSize) UNAVAILABLE diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp new file mode 100644 index 0000000000000..89c10c934b21c --- /dev/null +++ b/src/aotcompile.cpp @@ -0,0 +1,908 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "llvm-version.h" +#include "platform.h" +#include "options.h" + +// target support +#include +#include +#include +#include +#include +#include + +// analysis passes +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(JL_ASAN_ENABLED) +#include +#endif +#include +#include +#include +#if defined(USE_POLLY) +#include +#include +#include +#if defined(USE_POLLY_ACC) +#include +#endif +#endif + +// for outputting assembly +#include +#include +#include "llvm/Object/ArchiveWriter.h" +#include +#include +#include +#include +#include +#include +#include +#include +#if JL_LLVM_VERSION >= 100000 +#include +#endif + +#include +#include + + +using namespace llvm; + +// our passes +namespace llvm { + extern Pass *createLowerSimdLoopPass(); +} + +#if JL_LLVM_VERSION < 100000 +static const TargetMachine::CodeGenFileType CGFT_ObjectFile = TargetMachine::CGFT_ObjectFile; +#endif + + +#include "julia.h" +#include "julia_internal.h" +#include "jitlayers.h" +#include "julia_assert.h" + +// MSVC's link.exe requires each function declaration to have a Comdat section +// So rather than litter the code with conditionals, +// all global values that get emitted call this function +// and it decides whether the definition needs a Comdat section and adds the appropriate declaration +template // for GlobalObject's +static T *addComdat(T *G) +{ +#if defined(_OS_WINDOWS_) + if (!G->isDeclaration()) { + // Add comdat information to make MSVC link.exe happy + // it's valid to emit this for ld.exe too, + // but makes it very slow to link for no benefit +#if defined(_COMPILER_MICROSOFT_) + Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); + // ELF only supports Comdat::Any + jl_Comdat->setSelectionKind(Comdat::NoDuplicates); + G->setComdat(jl_Comdat); +#endif + // add __declspec(dllexport) to everything marked for export + if (G->getLinkage() == GlobalValue::ExternalLinkage) + G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); + else + G->setDLLStorageClass(GlobalValue::DefaultStorageClass); + } +#endif + return G; +} + + +typedef struct { + std::unique_ptr M; + std::vector jl_sysimg_fvars; + std::vector jl_sysimg_gvars; + std::map> jl_fvar_map; + std::map jl_value_to_llvm; // uses 1-based indexing +} jl_native_code_desc_t; + +extern "C" JL_DLLEXPORT +void jl_get_function_id(void *native_code, jl_code_instance_t *codeinst, + int32_t *func_idx, int32_t *specfunc_idx) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) { + // get the function index in the fvar lookup table + auto it = data->jl_fvar_map.find(codeinst); + if (it != data->jl_fvar_map.end()) { + std::tie(*func_idx, *specfunc_idx) = it->second; + } + } +} + +extern "C" +int32_t jl_get_llvm_gv(void *native_code, jl_value_t *p) +{ + // map a jl_value_t memory location to a GlobalVariable + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) { + auto it = data->jl_value_to_llvm.find(p); + if (it != data->jl_value_to_llvm.end()) { + return it->second; + } + } + return 0; +} + +extern "C" JL_DLLEXPORT +Module* jl_get_llvm_module(void *native_code) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) + return data->M.get(); + else + return NULL; +} + +extern "C" JL_DLLEXPORT +GlobalValue* jl_get_llvm_function(void *native_code, uint32_t idx) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) + return data->jl_sysimg_fvars[idx]; + else + return NULL; +} + +extern "C" JL_DLLEXPORT +LLVMContext* jl_get_llvm_context(void *native_code) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (data) + return &data->M->getContext(); + else + return NULL; +} + + +static void emit_offset_table(Module &mod, const std::vector &vars, StringRef name, Type *T_psize) +{ + // Emit a global variable with all the variable addresses. + // The cloning pass will convert them into offsets. + assert(!vars.empty()); + size_t nvars = vars.size(); + std::vector addrs(nvars); + for (size_t i = 0; i < nvars; i++) { + Constant *var = vars[i]; + addrs[i] = ConstantExpr::getBitCast(var, T_psize); + } + ArrayType *vars_type = ArrayType::get(T_psize, nvars); + new GlobalVariable(mod, vars_type, true, + GlobalVariable::ExternalLinkage, + ConstantArray::get(vars_type, addrs), + name); +} + +static bool is_safe_char(unsigned char c) +{ + return ('0' <= c && c <= '9') || + ('A' <= c && c <= 'Z') || + ('a' <= c && c <= 'z') || + (c == '_' || c == '$') || + (c >= 128 && c < 255); +} + +static const char hexchars[16] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + +static const char *const common_names[256] = { +// 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x00 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x10 + "SP", "NOT", "DQT", "YY", 0, "REM", "AND", "SQT", // 0x20 + "LPR", "RPR", "MUL", "SUM", 0, "SUB", "DOT", "DIV", // 0x28 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "COL", 0, "LT", "EQ", "GT", "QQ", // 0x30 + "AT", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "LBR", "RDV", "RBR", "POW", 0, // 0x50 + "TIC", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "LCR", "OR", "RCR", "TLD", "DEL", // 0x70 + 0 }; // remainder is filled with zeros, though are also all safe characters + +// reversibly removes special characters from the name of GlobalObjects, +// which might cause them to be treated special by LLVM or the system linker +// the only non-identifier characters we allow to appear are '.' and '$', +// and all of UTF-8 above code-point 128 (except 255) +// most are given "friendly" abbreviations +// the remaining few will print as hex +// e.g. mangles "llvm.a≠a$a!a##" as "llvmDOT.a≠a$aNOT.aYY.YY." +static void makeSafeName(GlobalObject &G) +{ + StringRef Name = G.getName(); + SmallVector SafeName; + for (unsigned char c : Name.bytes()) { + if (is_safe_char(c)) { + SafeName.push_back(c); + } + else { + if (common_names[c]) { + SafeName.push_back(common_names[c][0]); + SafeName.push_back(common_names[c][1]); + if (common_names[c][2]) + SafeName.push_back(common_names[c][2]); + } + else { + SafeName.push_back(hexchars[(c >> 4) & 0xF]); + SafeName.push_back(hexchars[c & 0xF]); + } + SafeName.push_back('.'); + } + } + if (SafeName.size() != Name.size()) + G.setName(StringRef(SafeName.data(), SafeName.size())); +} + + +// takes the running content that has collected in the shadow module and dump it to disk +// this builds the object file portion of the sysimage files for fast startup +extern "C" JL_DLLEXPORT +void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams) +{ + jl_native_code_desc_t *data = new jl_native_code_desc_t; + jl_codegen_params_t params; + params.params = &cgparams; + std::map emitted; + jl_method_instance_t *mi = NULL; + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + JL_LOCK(&codegen_lock); + + // compile all methods for the current world and type-inference world + size_t compile_for[] = { jl_typeinf_world, jl_world_counter }; + for (int worlds = 0; worlds < 2; worlds++) { + params.world = compile_for[worlds]; + if (!params.world) + continue; + size_t i, l; + for (i = 0, l = jl_array_len(methods); i < l; i++) { + mi = (jl_method_instance_t*)jl_array_ptr_ref(methods, i); + src = NULL; + // if this method is generally visible to the current compilation world, + // and this is either the primary world, or not applicable in the primary world + // then we want to compile and emit this + if (mi->def.method->primary_world <= params.world && params.world <= mi->def.method->deleted_world) { + // find and prepare the source code to compile + jl_value_t *ci = jl_rettype_inferred(mi, params.world, params.world); + jl_code_instance_t *codeinst = NULL; + if (ci != jl_nothing) { + codeinst = (jl_code_instance_t*)ci; + src = (jl_code_info_t*)codeinst->inferred; + jl_method_t *def = codeinst->def->def.method; + if ((jl_value_t*)src == jl_nothing) + src = NULL; + if (src && jl_is_method(def)) + src = jl_uncompress_ast(def, codeinst, (jl_array_t*)src); + } + if (src == NULL || !jl_is_code_info(src)) { + src = jl_type_infer(mi, params.world, 0); + codeinst = jl_get_method_inferred(mi, src->rettype, src->min_world, src->max_world); + if (src->inferred && !codeinst->inferred) + codeinst->inferred = jl_nothing; + } + if (src && !emitted.count(codeinst)) { + // now add it to our compilation results + JL_GC_PROMISE_ROOTED(codeinst->rettype); + jl_compile_result_t result = jl_emit_code(mi, src, codeinst->rettype, params); + if (std::get<0>(result)) + emitted[codeinst] = std::move(result); + } + } + } + // finally, make sure all referenced methods also get compiled or fixed up + jl_compile_workqueue(emitted, params); + } + JL_GC_POP(); + + // process the globals array, before jl_merge_module destroys them + std::vector gvars; + for (auto &global : params.globals) { + gvars.push_back(global.second->getName()); + data->jl_value_to_llvm[global.first] = gvars.size(); + } + + // clones the contents of the module `m` to the shadow_output collector + // while examining and recording what kind of function pointer we have + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(*shadow_output, VMap)); + for (auto &def : emitted) { + jl_merge_module(clone.get(), std::move(std::get<0>(def.second))); + jl_code_instance_t *this_code = def.first; + jl_llvm_functions_t decls = std::get<1>(def.second); + StringRef func = decls.functionObject; + StringRef cfunc = decls.specFunctionObject; + uint32_t func_id = 0; + uint32_t cfunc_id = 0; + if (func == "jl_fptr_args") { + func_id = -1; + } + else if (func == "jl_fptr_sparam") { + func_id = -2; + } + else { + data->jl_sysimg_fvars.push_back(cast(clone->getNamedValue(func))); + func_id = data->jl_sysimg_fvars.size(); + } + if (!cfunc.empty()) { + data->jl_sysimg_fvars.push_back(cast(clone->getNamedValue(cfunc))); + cfunc_id = data->jl_sysimg_fvars.size(); + } + data->jl_fvar_map[this_code] = std::make_tuple(func_id, cfunc_id); + } + + // now get references to the globals in the merged module + // and set them to be internalized and initialized at startup + for (auto &global : gvars) { + GlobalVariable *G = cast(clone->getNamedValue(global)); + G->setInitializer(ConstantPointerNull::get(cast(G->getValueType()))); + G->setLinkage(GlobalVariable::InternalLinkage); + data->jl_sysimg_gvars.push_back(G); + } + +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) + // setting the function personality enables stack unwinding and catching exceptions + // so make sure everything has something set + Type *T_int32 = Type::getInt32Ty(clone->getContext()); + Function *juliapersonality_func = + Function::Create(FunctionType::get(T_int32, true), + Function::ExternalLinkage, "__julia_personality", clone.get()); + juliapersonality_func->setDLLStorageClass(GlobalValue::DLLImportStorageClass); +#endif + + // move everything inside, now that we've merged everything + // (before adding the exported headers) + for (GlobalObject &G : clone->global_objects()) { + if (!G.isDeclaration()) { + G.setLinkage(Function::InternalLinkage); + makeSafeName(G); + addComdat(&G); +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) + // Add unwind exception personalities to functions to handle async exceptions + if (Function *F = dyn_cast(&G)) + F->setPersonalityFn(juliapersonality_func); +#endif + } + } + + data->M = std::move(clone); + + JL_UNLOCK(&codegen_lock); // Might GC + return (void*)data; +} + + +static void emit_result(std::vector &Archive, SmallVectorImpl &OS, + StringRef Name, std::vector &outputs) +{ + outputs.push_back({ OS.data(), OS.size() }); + Archive.push_back(NewArchiveMember(MemoryBufferRef(outputs.back(), Name))); + OS.clear(); +} + +static object::Archive::Kind getDefaultForHost(Triple &triple) +{ + if (triple.isOSDarwin()) + return object::Archive::K_DARWIN; + return object::Archive::K_GNU; +} + +typedef Error ArchiveWriterError; +static void reportWriterError(const ErrorInfoBase &E) +{ + std::string err = E.message(); + jl_safe_printf("ERROR: failed to emit output file %s\n", err.c_str()); +} + + +// takes the running content that has collected in the shadow module and dump it to disk +// this builds the object file portion of the sysimage files for fast startup +extern "C" +void jl_dump_native(void *native_code, + const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, + const char *sysimg_data, size_t sysimg_len) +{ + JL_TIMING(NATIVE_DUMP); + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + LLVMContext &Context = data->M->getContext(); + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); + // make sure to emit the native object format, even if FORCE_ELF was set in codegen +#if defined(_OS_WINDOWS_) + TheTriple.setObjectFormat(Triple::COFF); +#elif defined(_OS_DARWIN_) + TheTriple.setObjectFormat(Triple::MachO); + TheTriple.setOS(llvm::Triple::MacOSX); +#endif + std::unique_ptr TM( + jl_TargetMachine->getTarget().createTargetMachine( + TheTriple.getTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + Reloc::PIC_, +#else + Optional(), +#endif +#if defined(_CPU_PPC_) || defined(_CPU_PPC64_) + // On PPC the small model is limited to 16bit offsets + CodeModel::Medium, +#else + // Use small model so that we can use signed 32bits offset in the function and GV tables + CodeModel::Small, +#endif + CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? + )); + + legacy::PassManager PM; + addTargetPasses(&PM, TM.get()); + + // set up optimization passes + SmallVector bc_Buffer; + SmallVector obj_Buffer; + SmallVector unopt_bc_Buffer; + raw_svector_ostream bc_OS(bc_Buffer); + raw_svector_ostream obj_OS(obj_Buffer); + raw_svector_ostream unopt_bc_OS(unopt_bc_Buffer); + std::vector bc_Archive; + std::vector obj_Archive; + std::vector unopt_bc_Archive; + std::vector outputs; + + if (unopt_bc_fname) + PM.add(createBitcodeWriterPass(unopt_bc_OS)); + if (bc_fname || obj_fname) + addOptimizationPasses(&PM, jl_options.opt_level, true, true); + if (bc_fname) + PM.add(createBitcodeWriterPass(bc_OS)); + if (obj_fname) + if (TM->addPassesToEmitFile(PM, obj_OS, nullptr, CGFT_ObjectFile, false)) + jl_safe_printf("ERROR: target does not support generation of object files\n"); + + // Reset the target triple to make sure it matches the new target machine + data->M->setTargetTriple(TM->getTargetTriple().str()); + DataLayout DL = TM->createDataLayout(); + DL.reset(DL.getStringRepresentation() + "-ni:10:11:12:13"); + data->M->setDataLayout(DL); + Type *T_size; + if (sizeof(size_t) == 8) + T_size = Type::getInt64Ty(Context); + else + T_size = Type::getInt32Ty(Context); + Type *T_psize = T_size->getPointerTo(); + + // add metadata information + if (imaging_mode) { + emit_offset_table(*data->M, data->jl_sysimg_gvars, "jl_sysimg_gvars", T_psize); + emit_offset_table(*data->M, data->jl_sysimg_fvars, "jl_sysimg_fvars", T_psize); + + // reflect the address of the jl_RTLD_DEFAULT_handle variable + // back to the caller, so that we can check for consistency issues + GlobalValue *jlRTLD_DEFAULT_var = data->M->getNamedValue("jl_RTLD_DEFAULT_handle"); + addComdat(new GlobalVariable(*data->M, + jlRTLD_DEFAULT_var->getType(), + true, + GlobalVariable::ExternalLinkage, + jlRTLD_DEFAULT_var, + "jl_RTLD_DEFAULT_handle_pointer")); + } + + // do the actual work + auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name) { + PM.run(M); + if (unopt_bc_fname) + emit_result(unopt_bc_Archive, unopt_bc_Buffer, unopt_bc_Name, outputs); + if (bc_fname) + emit_result(bc_Archive, bc_Buffer, bc_Name, outputs); + if (obj_fname) + emit_result(obj_Archive, obj_Buffer, obj_Name, outputs); + }; + + add_output(*data->M, "unopt.bc", "text.bc", "text.o"); + + std::unique_ptr sysimage(new Module("sysimage", Context)); + sysimage->setTargetTriple(data->M->getTargetTriple()); + sysimage->setDataLayout(data->M->getDataLayout()); + data->M.reset(); // free memory for data->M + + addComdat(new GlobalVariable(*sysimage, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, globalUnique + 1), + "jl_globalUnique")); + + if (sysimg_data) { + Constant *data = ConstantDataArray::get(Context, + ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); + addComdat(new GlobalVariable(*sysimage, data->getType(), false, + GlobalVariable::ExternalLinkage, + data, "jl_system_image_data"))->setAlignment(Align(64)); + Constant *len = ConstantInt::get(T_size, sysimg_len); + addComdat(new GlobalVariable(*sysimage, len->getType(), true, + GlobalVariable::ExternalLinkage, + len, "jl_system_image_size")); + } + add_output(*sysimage, "data.bc", "data.bc", "data.o"); + + object::Archive::Kind Kind = getDefaultForHost(TheTriple); + if (unopt_bc_fname) + handleAllErrors(writeArchive(unopt_bc_fname, unopt_bc_Archive, true, + Kind, true, false), reportWriterError); + if (bc_fname) + handleAllErrors(writeArchive(bc_fname, bc_Archive, true, + Kind, true, false), reportWriterError); + if (obj_fname) + handleAllErrors(writeArchive(obj_fname, obj_Archive, true, + Kind, true, false), reportWriterError); + + delete data; +} + +// clones the contents of the module `m` to the shadow_output collector +// TODO: this should be deprecated +void jl_add_to_shadow(Module *m) +{ + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(*m, VMap)); + jl_merge_module(shadow_output, std::move(clone)); +} + + +void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM) +{ + PM->add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); + PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); +} + +// this defines the set of optimization passes defined for Julia at various optimization levels. +// it assumes that the TLI and TTI wrapper passes have already been added. +void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, + bool lower_intrinsics, bool dump_native) +{ +#ifdef JL_DEBUG_BUILD + PM->add(createGCInvariantVerifierPass(true)); + PM->add(createVerifierPass()); +#endif + +#if defined(JL_ASAN_ENABLED) + PM->add(createAddressSanitizerFunctionPass()); +#endif +#if defined(JL_MSAN_ENABLED) + PM->add(llvm::createMemorySanitizerPass(true)); +#endif + if (opt_level < 2) { + PM->add(createCFGSimplificationPass()); // Clean up disgusting code + if (opt_level == 1) { + PM->add(createSROAPass()); // Break up aggregate allocas + PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. + PM->add(createEarlyCSEPass()); + } + PM->add(createMemCpyOptPass()); // Remove memcpy / form memset + PM->add(createAlwaysInlinerLegacyPass()); // Respect always_inline + if (lower_intrinsics) { + PM->add(createBarrierNoopPass()); + PM->add(createLowerExcHandlersPass()); + PM->add(createGCInvariantVerifierPass(false)); + PM->add(createLateLowerGCFramePass()); + PM->add(createFinalLowerGCPass()); + PM->add(createLowerPTLSPass(dump_native)); + } + PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop + if (dump_native) + PM->add(createMultiVersioningPass()); + return; + } + PM->add(createPropagateJuliaAddrspaces()); + PM->add(createScopedNoAliasAAWrapperPass()); + PM->add(createTypeBasedAAWrapperPass()); + if (opt_level >= 3) { + PM->add(createBasicAAWrapperPass()); + } + // list of passes from vmkit + PM->add(createCFGSimplificationPass()); // Clean up disgusting code + PM->add(createDeadCodeEliminationPass()); + PM->add(createSROAPass()); // Kill useless allocas + + PM->add(createMemCpyOptPass()); + + PM->add(createAlwaysInlinerLegacyPass()); // Respect always_inline + + // Running `memcpyopt` between this and `sroa` seems to give `sroa` a hard time + // merging the `alloca` for the unboxed data and the `alloca` created by the `alloc_opt` + // pass. + PM->add(createAllocOptPass()); + PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. + // Now that SROA has cleaned up for front-end mess, a lot of control flow should + // be more evident - try to clean it up. + PM->add(createCFGSimplificationPass()); // Merge & remove BBs + if (dump_native) + PM->add(createMultiVersioningPass()); + PM->add(createSROAPass()); // Break up aggregate allocas + PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. + PM->add(createJumpThreadingPass()); // Thread jumps. + PM->add(createInstructionCombiningPass()); // Combine silly seq's + + //PM->add(createCFGSimplificationPass()); // Merge & remove BBs + PM->add(createReassociatePass()); // Reassociate expressions + + // this has the potential to make some things a bit slower + //PM->add(createBBVectorizePass()); + + PM->add(createEarlyCSEPass()); //// **** + + // Load forwarding above can expose allocations that aren't actually used + // remove those before optimizing loops. + PM->add(createAllocOptPass()); + PM->add(createLoopIdiomPass()); //// **** + PM->add(createLoopRotatePass()); // Rotate loops. +#ifdef USE_POLLY + // LCSSA (which has already run at this point due to the dependencies of the + // above passes) introduces redundant phis that hinder Polly. Therefore we + // run InstCombine here to remove them. + PM->add(createInstructionCombiningPass()); + PM->add(polly::createCodePreparationPass()); + polly::registerPollyPasses(*PM); + PM->add(polly::createCodegenCleanupPass()); +#endif + // LoopRotate strips metadata from terminator, so run LowerSIMD afterwards + PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop + PM->add(createLICMPass()); // Hoist loop invariants + PM->add(createLoopUnswitchPass()); // Unswitch loops. + // Subsequent passes not stripping metadata from terminator + PM->add(createInstructionCombiningPass()); + PM->add(createIndVarSimplifyPass()); // Canonicalize indvars + PM->add(createLoopDeletionPass()); // Delete dead loops + PM->add(createSimpleLoopUnrollPass()); // Unroll small loops + //PM->add(createLoopStrengthReducePass()); // (jwb added) + + // Run our own SROA on heap objects before LLVM's + PM->add(createAllocOptPass()); + // Re-run SROA after loop-unrolling (useful for small loops that operate, + // over the structure of an aggregate) + PM->add(createSROAPass()); // Break up aggregate allocas + PM->add(createInstructionCombiningPass()); // Clean up after the unroller + PM->add(createGVNPass()); // Remove redundancies + PM->add(createMemCpyOptPass()); // Remove memcpy / form memset + PM->add(createSCCPPass()); // Constant prop with SCCP + + // Run instcombine after redundancy elimination to exploit opportunities + // opened up by them. + PM->add(createInstructionCombiningPass()); + PM->add(createJumpThreadingPass()); // Thread jumps + PM->add(createDeadStoreEliminationPass()); // Delete dead stores + + // More dead allocation (store) deletion before loop optimization + PM->add(createAllocOptPass()); + // see if all of the constant folding has exposed more loops + // to simplification and deletion + // this helps significantly with cleaning up iteration + PM->add(createCFGSimplificationPass()); // Merge & remove BBs + PM->add(createLoopIdiomPass()); + PM->add(createLoopDeletionPass()); // Delete dead loops + PM->add(createJumpThreadingPass()); // Thread jumps + + PM->add(createSLPVectorizerPass()); // Vectorize straight-line code + PM->add(createAggressiveDCEPass()); // Delete dead instructions + PM->add(createInstructionCombiningPass()); // Clean up after SLP loop vectorizer + PM->add(createLoopVectorizePass()); // Vectorize loops + PM->add(createInstructionCombiningPass()); // Clean up after loop vectorizer + + if (lower_intrinsics) { + // LowerPTLS removes an indirect call. As a result, it is likely to trigger + // LLVM's devirtualization heuristics, which would result in the entire + // pass pipeline being re-exectuted. Prevent this by inserting a barrier. + PM->add(createBarrierNoopPass()); + PM->add(createLowerExcHandlersPass()); + PM->add(createGCInvariantVerifierPass(false)); + PM->add(createLateLowerGCFramePass()); + PM->add(createFinalLowerGCPass()); + // Remove dead use of ptls + PM->add(createDeadCodeEliminationPass()); + PM->add(createLowerPTLSPass(dump_native)); + // Clean up write barrier and ptls lowering + PM->add(createCFGSimplificationPass()); + } + PM->add(createCombineMulAddPass()); +} + +// An LLVM module pass that just runs all julia passes in order. Useful for +// debugging +template +class JuliaPipeline : public Pass { +public: + static char ID; + // A bit of a hack, but works + struct TPMAdapter : public PassManagerBase { + PMTopLevelManager *TPM; + TPMAdapter(PMTopLevelManager *TPM) : TPM(TPM) {} + void add(Pass *P) { TPM->schedulePass(P); } + }; + void preparePassManager(PMStack &Stack) override { + (void)jl_init_llvm(); + PMTopLevelManager *TPM = Stack.top()->getTopLevelManager(); + TPMAdapter Adapter(TPM); + addTargetPasses(&Adapter, jl_TargetMachine); + addOptimizationPasses(&Adapter, OptLevel); + } + JuliaPipeline() : Pass(PT_PassManager, ID) {} + Pass *createPrinterPass(raw_ostream &O, const std::string &Banner) const override { + return createPrintModulePass(O, Banner); + } +}; +template<> char JuliaPipeline<0>::ID = 0; +template<> char JuliaPipeline<2>::ID = 0; +template<> char JuliaPipeline<3>::ID = 0; +static RegisterPass> X("juliaO0", "Runs the entire julia pipeline (at -O0)", false, false); +static RegisterPass> Y("julia", "Runs the entire julia pipeline (at -O2)", false, false); +static RegisterPass> Z("juliaO3", "Runs the entire julia pipeline (at -O3)", false, false); + +extern "C" JL_DLLEXPORT +void jl_add_optimization_passes(LLVMPassManagerRef PM, int opt_level, int lower_intrinsics) { + addOptimizationPasses(unwrap(PM), opt_level, lower_intrinsics); +} + +// --- native code info, and dump function to IR and ASM --- +// Get pointer to llvm::Function instance, compiling if necessary +// for use in reflection from Julia. +// this is paired with jl_dump_function_ir, jl_dump_method_asm, jl_dump_llvm_asm in particular ways: +// misuse will leak memory or cause read-after-free +extern "C" JL_DLLEXPORT +void *jl_get_llvmf_defn(jl_method_instance_t *mi, size_t world, char getwrapper, char optimize, const jl_cgparams_t params) +{ + if (jl_is_method(mi->def.method) && mi->def.method->source == NULL && + mi->def.method->generator == NULL) { + // not a generic function + return NULL; + } + + static legacy::PassManager *PM; + if (!PM) { + PM = new legacy::PassManager(); + addTargetPasses(PM, jl_TargetMachine); + addOptimizationPasses(PM, jl_options.opt_level); + } + + // get the source code for this function + jl_value_t *jlrettype = (jl_value_t*)jl_any_type; + jl_code_info_t *src = NULL; + JL_GC_PUSH2(&src, &jlrettype); + jl_value_t *ci = jl_rettype_inferred(mi, world, world); + if (ci != jl_nothing) { + jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; + src = (jl_code_info_t*)codeinst->inferred; + if ((jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method)) + src = jl_uncompress_ast(mi->def.method, codeinst, (jl_array_t*)src); + jlrettype = codeinst->rettype; + } + if (!src || (jl_value_t*)src == jl_nothing) { + src = jl_type_infer(mi, world, 0); + if (src) + jlrettype = src->rettype; + else if (jl_is_method(mi->def.method)) { + src = mi->def.method->generator ? jl_code_for_staged(mi) : (jl_code_info_t*)mi->def.method->source; + if (src && !jl_is_code_info(src) && jl_is_method(mi->def.method)) + src = jl_uncompress_ast(mi->def.method, NULL, (jl_array_t*)src); + } + // TODO: use mi->uninferred + } + + // emit this function into a new llvm module + if (src && jl_is_code_info(src)) { + jl_codegen_params_t output; + output.world = world; + output.params = ¶ms; + std::unique_ptr m; + jl_llvm_functions_t decls; + JL_LOCK(&codegen_lock); + std::tie(m, decls) = jl_emit_code(mi, src, jlrettype, output); + + Function *F = NULL; + if (m) { + // if compilation succeeded, prepare to return the result + if (optimize) + PM->run(*m.get()); + const std::string *fname; + if (decls.functionObject == "jl_fptr_args" || decls.functionObject == "jl_fptr_sparam") + getwrapper = false; + if (!getwrapper) + fname = &decls.specFunctionObject; + else + fname = &decls.functionObject; + F = cast(m->getNamedValue(*fname)); + m.release(); // the return object `llvmf` will be the owning pointer + } + JL_GC_POP(); + JL_UNLOCK(&codegen_lock); // Might GC + if (F) + return F; + } + + const char *mname = name_from_method_instance(mi); + jl_errorf("unable to compile source for function %s", mname); +} + +/// addPassesToX helper drives creation and initialization of TargetPassConfig. +static MCContext * +addPassesToGenerateCode(LLVMTargetMachine *TM, PassManagerBase &PM) { + TargetPassConfig *PassConfig = TM->createPassConfig(PM); + PassConfig->setDisableVerify(false); + PM.add(PassConfig); + MachineModuleInfo *MMI = new MachineModuleInfo(TM); + PM.add(MMI); + if (PassConfig->addISelPasses()) + return NULL; + PassConfig->addMachinePasses(); + PassConfig->setInitialized(); + return &MMI->getContext(); +} + +void jl_strip_llvm_debug(Module *m); + + +// get a native assembly for llvm::Function +// TODO: implement debuginfo handling +extern "C" JL_DLLEXPORT +jl_value_t *jl_dump_llvm_asm(void *F, const char* asm_variant, const char *debuginfo) +{ + // precise printing via IR assembler + SmallVector ObjBufferSV; + { // scope block + Function *f = (Function*)F; + llvm::raw_svector_ostream asmfile(ObjBufferSV); + assert(!f->isDeclaration()); + std::unique_ptr m(f->getParent()); + for (auto &f2 : m->functions()) { + if (f != &f2 && !f->isDeclaration()) + f2.deleteBody(); + } + jl_strip_llvm_debug(m.get()); + legacy::PassManager PM; + LLVMTargetMachine *TM = static_cast(jl_TargetMachine); + MCContext *Context = addPassesToGenerateCode(TM, PM); + if (Context) { + const MCSubtargetInfo &STI = *TM->getMCSubtargetInfo(); + const MCAsmInfo &MAI = *TM->getMCAsmInfo(); + const MCRegisterInfo &MRI = *TM->getMCRegisterInfo(); + const MCInstrInfo &MII = *TM->getMCInstrInfo(); + unsigned OutputAsmDialect = MAI.getAssemblerDialect(); + if (!strcmp(asm_variant, "att")) + OutputAsmDialect = 0; + if (!strcmp(asm_variant, "intel")) + OutputAsmDialect = 1; + MCInstPrinter *InstPrinter = TM->getTarget().createMCInstPrinter( + TM->getTargetTriple(), OutputAsmDialect, MAI, MII, MRI); + std::unique_ptr MAB(TM->getTarget().createMCAsmBackend( + STI, MRI, TM->Options.MCOptions)); + std::unique_ptr MCE; + auto FOut = llvm::make_unique(asmfile); + std::unique_ptr S(TM->getTarget().createAsmStreamer( + *Context, std::move(FOut), true, + true, InstPrinter, + std::move(MCE), std::move(MAB), + false)); + std::unique_ptr Printer( + TM->getTarget().createAsmPrinter(*TM, std::move(S))); + if (Printer) { + PM.add(Printer.release()); + PM.run(*m); + } + } + } + return jl_pchar_to_string(ObjBufferSV.data(), ObjBufferSV.size()); +} diff --git a/src/ccall.cpp b/src/ccall.cpp index 4bd5e4624c0e5..2dd5462c58701 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -42,6 +42,17 @@ lazyModule(Func &&func) std::forward(func)); } + +void copy_to_shadow(GlobalVariable *gv) +{ + // hack: make a copy of all globals in the shadow_output + if (!imaging_mode) + return; + GlobalVariable *shadowvar = global_proto(gv, shadow_output); + shadowvar->setInitializer(gv->getInitializer()); + shadowvar->setLinkage(GlobalVariable::InternalLinkage); +} + // Find or create the GVs for the library and symbol lookup. // Return `runtime_lib` (whether the library name is a string) // The `lib` and `sym` GV returned may not be in the current module. @@ -77,6 +88,7 @@ static bool runtime_sym_gvs(const char *f_lib, const char *f_name, MT &&M, libptrgv = new GlobalVariable(*M, T_pint8, false, GlobalVariable::ExternalLinkage, Constant::getNullValue(T_pint8), name); + copy_to_shadow(libptrgv); libgv.first = global_proto(libptrgv); } else { @@ -96,6 +108,7 @@ static bool runtime_sym_gvs(const char *f_lib, const char *f_name, MT &&M, llvmgv = new GlobalVariable(*M, T_pvoidfunc, false, GlobalVariable::ExternalLinkage, Constant::getNullValue(T_pvoidfunc), name); + copy_to_shadow(llvmgv); llvmgv = global_proto(llvmgv); } @@ -186,6 +199,8 @@ static DenseMap,GlobalVariable*>> allPltMap; +void jl_add_to_shadow(Module *m); + // Emit a "PLT" entry that will be lazily initialized // when being called the first time. static GlobalVariable *emit_plt_thunk( @@ -204,7 +219,6 @@ static GlobalVariable *emit_plt_thunk( Function *plt = Function::Create(functype, GlobalVariable::ExternalLinkage, fname, M); - jl_init_function(plt); plt->setAttributes(attrs); if (cc != CallingConv::C) plt->setCallingConv(cc); @@ -250,7 +264,8 @@ static GlobalVariable *emit_plt_thunk( irbuilder.ClearInsertionPoint(); got = global_proto(got); // exchange got for the permanent global before jl_finalize_module destroys it - jl_finalize_module(M, true); + jl_add_to_shadow(M); + jl_finalize_module(std::unique_ptr(M)); auto shadowgot = cast(shadow_output->getNamedValue(gname)); @@ -496,8 +511,7 @@ static Value *julia_to_native( jl_codectx_t &ctx, Type *to, bool toboxed, jl_value_t *jlto, jl_unionall_t *jlto_env, const jl_cgval_t &jvinfo, - bool byRef, int argn, - bool *needStackRestore) + bool byRef, int argn) { // We're passing Any if (toboxed) { @@ -597,6 +611,24 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va } +static void jl_rethrow_with_add(const char *fmt, ...) +{ + jl_value_t *exc = jl_current_exception(); + if (jl_typeis(exc, jl_errorexception_type)) { + char *str = jl_string_data(jl_fieldref(exc, 0)); + char buf[1024]; + va_list args; + va_start(args, fmt); + int nc = vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + nc += snprintf(buf + nc, sizeof(buf) - nc, ": %s", str); + jl_value_t *msg = jl_pchar_to_string(buf, nc); + JL_GC_PUSH1(&msg); + jl_throw(jl_new_struct(jl_errorexception_type, msg)); + } + jl_rethrow(); +} + static jl_value_t* try_eval(jl_codectx_t &ctx, jl_value_t *ex, const char *failure) { jl_value_t *constant = static_eval(ctx, ex, true, true); @@ -647,7 +679,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg rt = (jl_value_t*)jl_voidpointer_type; } Type *lrt = T_size; - assert(lrt == julia_type_to_llvm(rt)); + assert(lrt == julia_type_to_llvm(ctx, rt)); interpret_symbol_arg(ctx, sym, args[1], "cglobal", false); @@ -835,14 +867,14 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar if (jl_is_ssavalue(ir_arg)) ir_arg = jl_arrayref((jl_array_t*)ctx.source->code, ((jl_ssavalue_t*)ir_arg)->id - 1); ir = try_eval(ctx, ir_arg, "error statically evaluating llvm IR argument"); - if (jl_is_ssavalue(args[2])) { + if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) { jl_value_t *rtt = jl_arrayref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[2])->id - 1); if (jl_is_type_type(rtt)) rt = jl_tparam0(rtt); } if (rt == NULL) rt = try_eval(ctx, args[2], "error statically evaluating llvmcall return type"); - if (jl_is_ssavalue(args[3])) { + if (jl_is_ssavalue(args[3]) && !jl_is_long(ctx.source->ssavaluetypes)) { jl_value_t *att = jl_arrayref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[3])->id - 1); if (jl_is_type_type(att)) at = jl_tparam0(att); @@ -887,7 +919,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar for (size_t i = 0; i < nargt; ++i) { jl_value_t *tti = jl_svecref(tt,i); bool toboxed; - Type *t = julia_type_to_llvm(tti, &toboxed); + Type *t = julia_type_to_llvm(ctx, tti, &toboxed); argtypes.push_back(t); if (4 + i > nargs) { jl_error("Missing arguments to llvmcall!"); @@ -895,18 +927,18 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar jl_value_t *argi = args[4 + i]; jl_cgval_t arg = emit_expr(ctx, argi); - Value *v = julia_to_native(ctx, t, toboxed, tti, NULL, arg, false, i, NULL); + Value *v = julia_to_native(ctx, t, toboxed, tti, NULL, arg, false, i); bool issigned = jl_signed_type && jl_subtype(tti, (jl_value_t*)jl_signed_type); argvals[i] = llvm_type_rewrite(ctx, v, t, issigned); } Function *f; bool retboxed; - Type *rettype = julia_type_to_llvm(rtt, &retboxed); + Type *rettype = julia_type_to_llvm(ctx, rtt, &retboxed); if (isString) { // Make sure to find a unique name std::string ir_name; - while(true) { + while (true) { std::stringstream name; name << (ctx.f->getName().str()) << "u" << i++; ir_name = name.str(); @@ -998,11 +1030,9 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar std::stringstream name; name << "jl_llvmcall" << llvmcallnumbering++; f->setName(name.str()); - jl_init_function(f); f = cast(prepare_call(function_proto(f))); } else { - jl_init_function(f); f->setLinkage(GlobalValue::LinkOnceODRLinkage); } @@ -1066,12 +1096,14 @@ class function_sig_t { jl_unionall_t *unionall_env; // UnionAll environment for `at` and `rt` size_t nccallargs; // number of actual arguments size_t nreqargs; // number of required arguments in ccall function definition + jl_codegen_params_t *ctx; - function_sig_t(const char *fname, Type *lrt, jl_value_t *rt, bool retboxed, jl_svec_t *at, jl_unionall_t *unionall_env, size_t nreqargs, CallingConv::ID cc, bool llvmcall) + function_sig_t(const char *fname, Type *lrt, jl_value_t *rt, bool retboxed, jl_svec_t *at, jl_unionall_t *unionall_env, size_t nreqargs, CallingConv::ID cc, bool llvmcall, jl_codegen_params_t *ctx) : lrt(lrt), retboxed(retboxed), prt(NULL), sret(0), cc(cc), llvmcall(llvmcall), at(at), rt(rt), unionall_env(unionall_env), - nccallargs(jl_svec_len(at)), nreqargs(nreqargs) + nccallargs(jl_svec_len(at)), nreqargs(nreqargs), + ctx(ctx) { err_msg = generate_func_sig(fname); } @@ -1159,7 +1191,7 @@ std::string generate_func_sig(const char *fname) } } - t = julia_struct_to_llvm(tti, unionall_env, &isboxed, llvmcall); + t = _julia_struct_to_llvm(ctx, tti, unionall_env, &isboxed, llvmcall); if (t == NULL || t == T_void) { return make_errmsg(fname, i + 1, " doesn't correspond to a C type"); } @@ -1290,6 +1322,7 @@ static bool verify_ref_type(jl_codectx_t &ctx, jl_value_t* ref, jl_unionall_t *u static const std::string verify_ccall_sig(jl_value_t *&rt, jl_value_t *at, jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, + jl_codegen_params_t *ctx, Type *&lrt, bool &retboxed, bool &static_rt, bool llvmcall=false) { JL_TYPECHK(ccall, type, rt); @@ -1300,7 +1333,7 @@ static const std::string verify_ccall_sig(jl_value_t *&rt, jl_value_t *at, rt = (jl_value_t*)jl_any_type; } - lrt = julia_struct_to_llvm(rt, unionall_env, &retboxed, llvmcall); + lrt = _julia_struct_to_llvm(ctx, rt, unionall_env, &retboxed, llvmcall); if (lrt == NULL) return "return type doesn't correspond to a C type"; @@ -1420,6 +1453,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) /* inputs: */ rt, at, unionall, ctx.spvals_ptr == NULL ? ctx.linfo->sparam_vals : NULL, + &ctx.emission_context, /* outputs: */ lrt, retboxed, static_rt, /* optional arguments */ @@ -1445,7 +1479,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) jl_add_method_root(ctx, rt); function_sig_t sig("ccall", lrt, rt, retboxed, (jl_svec_t*)at, unionall, nreqargs, - cc, llvmcall); + cc, llvmcall, &ctx.emission_context); for (size_t i = 0; i < nccallargs; i++) { jl_value_t *tti = jl_svecref(at, i); if (jl_is_abstract_ref_type(tti)) { @@ -1480,7 +1514,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) isboxed = false; } else { - largty = julia_struct_to_llvm(tti, unionall, &isboxed, llvmcall); + largty = _julia_struct_to_llvm(&ctx.emission_context, tti, unionall, &isboxed, llvmcall); } if (isboxed) { ary = boxed(ctx, argv[0]); @@ -1723,11 +1757,6 @@ jl_cgval_t function_sig_t::emit_a_ccall( FunctionType *functype = this->functype(); - // save place before arguments, for possible insertion of temp arg area saving code. - BasicBlock::InstListType &instList = ctx.builder.GetInsertBlock()->getInstList(); - Instruction *savespot = instList.empty() ? NULL : &instList.back(); - - bool needStackRestore = false; Value **argvals = (Value**) alloca((nccallargs + sret) * sizeof(Value*)); for (size_t ai = 0; ai < nccallargs; ai++) { // Current C function parameter @@ -1758,8 +1787,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( jargty_in_env = (jl_value_t*)jl_voidpointer_type; } - v = julia_to_native(ctx, largty, toboxed, jargty_in_env, unionall_env, arg, byRef, - ai, &needStackRestore); + v = julia_to_native(ctx, largty, toboxed, jargty_in_env, unionall_env, arg, byRef, ai); bool issigned = jl_signed_type && jl_subtype(jargty, (jl_value_t*)jl_signed_type); if (byRef) { v = decay_derived(v); @@ -1800,18 +1828,6 @@ jl_cgval_t function_sig_t::emit_a_ccall( } } - Instruction *stacksave = NULL; - if (needStackRestore) { - stacksave = CallInst::Create(Intrinsic::getDeclaration(jl_Module, - Intrinsic::stacksave)); - if (savespot) { - instList.insertAfter(savespot->getIterator(), stacksave); - } - else { - instList.push_front(stacksave); - } - } - // make LLVM function object for the target // keep this close to the function call, so that the compiler can // optimize the global pointer load in the common case @@ -1851,13 +1867,12 @@ jl_cgval_t function_sig_t::emit_a_ccall( } else if (symarg.fptr != NULL) { Type *funcptype = PointerType::get(functype, 0); - llvmf = literal_static_pointer_val(ctx, (void*)(uintptr_t)symarg.fptr, funcptype); + llvmf = literal_static_pointer_val((void*)(uintptr_t)symarg.fptr, funcptype); if (imaging_mode) jl_printf(JL_STDERR,"WARNING: literal address used in ccall for %s; code cannot be statically compiled\n", symarg.f_name); } else { assert(symarg.f_name != NULL); - PointerType *funcptype = PointerType::get(functype, 0); if (imaging_mode) { // vararg requires musttail, @@ -1869,8 +1884,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } else { void *symaddr; - - void* libsym = jl_get_library_(symarg.f_lib, 0); + void *libsym = jl_get_library_(symarg.f_lib, 0); if (!libsym || !jl_dlsym(libsym, symarg.f_name, &symaddr, 0)) { // either the library or the symbol could not be found, place a runtime // lookup here instead. @@ -1878,7 +1892,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } else { // since we aren't saving this code, there's no sense in // putting anything complicated here: just JIT the function address - llvmf = literal_static_pointer_val(ctx, symaddr, funcptype); + llvmf = literal_static_pointer_val(symaddr, funcptype); } } } @@ -1894,10 +1908,6 @@ jl_cgval_t function_sig_t::emit_a_ccall( ((CallInst*)ret)->setCallingConv(cc); if (!sret) result = ret; - if (needStackRestore) { - assert(stacksave != NULL); - ctx.builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::stackrestore), stacksave); - } if (0) { // Enable this to turn on SSPREQ (-fstack-protector) on the function containing this ccall ctx.f->addFnAttr(Attribute::StackProtectReq); } @@ -1926,7 +1936,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( } } else { - Type *jlrt = julia_type_to_llvm(rt, &jlretboxed); // compute the real "julian" return type and compute whether it is boxed + Type *jlrt = julia_type_to_llvm(ctx, rt, &jlretboxed); // compute the real "julian" return type and compute whether it is boxed if (type_is_ghost(jlrt)) { return ghostValue(rt); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 188fc260c0ca7..824c6021fe837 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -198,59 +198,61 @@ Metadata *to_md_tree(jl_value_t *val) { // --- Debug info --- -static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed = false) +static DIType *_julia_type_to_di(jl_codegen_params_t *ctx, jl_value_t *jt, DIBuilder *dbuilder, bool isboxed) { - if (isboxed || !jl_is_datatype(jt)) - return jl_pvalue_dillvmt; jl_datatype_t *jdt = (jl_datatype_t*)jt; - // always return the boxed representation for types with hidden content - if (jdt->ditype != NULL) - return (DIType*)jdt->ditype; + if (isboxed || !jl_is_datatype(jt) || !jdt->isconcretetype) + return jl_pvalue_dillvmt; + assert(jdt->uid && jdt->layout); + DIType* _ditype = NULL; + DIType* &ditype = (ctx ? ctx->ditypes[jdt] : _ditype); + if (ditype) + return ditype; + const char *tname = jl_symbol_name(jdt->name->name); if (jl_is_primitivetype(jt)) { uint64_t SizeInBits = jl_datatype_nbits(jdt); - llvm::DIType *t = dbuilder->createBasicType( - jl_symbol_name(jdt->name->name), - SizeInBits, - llvm::dwarf::DW_ATE_unsigned); - jdt->ditype = t; - return t; + ditype = dbuilder->createBasicType(tname, SizeInBits, llvm::dwarf::DW_ATE_unsigned); } - if (jl_is_structtype(jt) && jdt->uid && jdt->layout && !jl_is_layout_opaque(jdt->layout)) { + else if (jl_is_structtype(jt) && !jl_is_layout_opaque(jdt->layout)) { size_t ntypes = jl_datatype_nfields(jdt); - const char *tname = jl_symbol_name(jdt->name->name); + std::vector Elements(ntypes); + for (unsigned i = 0; i < ntypes; i++) { + jl_value_t *el = jl_svecref(jdt->types, i); + DIType *di; + if (jl_field_isptr(jdt, i)) + di = jl_pvalue_dillvmt; + else + di = _julia_type_to_di(ctx, el, dbuilder, false); + Elements[i] = di; + } + DINodeArray ElemArray = dbuilder->getOrCreateArray(Elements); std::stringstream unique_name; unique_name << jdt->uid; - llvm::DICompositeType *ct = dbuilder->createStructType( + ditype = dbuilder->createStructType( NULL, // Scope tname, // Name NULL, // File 0, // LineNumber jl_datatype_nbits(jdt), // SizeInBits 8 * jl_datatype_align(jdt), // AlignInBits - DIFlagZero, // Flags + DINode::FlagZero, // Flags NULL, // DerivedFrom - DINodeArray(), // Elements + ElemArray, // Elements dwarf::DW_LANG_Julia, // RuntimeLanguage nullptr, // VTableHolder unique_name.str() // UniqueIdentifier ); - jdt->ditype = ct; - std::vector Elements; - for (unsigned i = 0; i < ntypes; i++) { - jl_value_t *el = jl_svecref(jdt->types, i); - DIType *di; - if (jl_field_isptr(jdt, i)) - di = jl_pvalue_dillvmt; - else - di = julia_type_to_di(el, dbuilder, false); - Elements.push_back(di); - } - dbuilder->replaceArrays(ct, dbuilder->getOrCreateArray(ArrayRef(Elements))); - return ct; } - jdt->ditype = dbuilder->createTypedef(jl_pvalue_dillvmt, - jl_symbol_name(jdt->name->name), NULL, 0, NULL); - return (llvm::DIType*)jdt->ditype; + else { + // return a typealias for types with hidden content + ditype = dbuilder->createTypedef(jl_pvalue_dillvmt, tname, NULL, 0, NULL); + } + return ditype; +} + +static DIType *julia_type_to_di(jl_codectx_t &ctx, jl_value_t *jt, DIBuilder *dbuilder, bool isboxed) +{ + return _julia_type_to_di(&ctx.emission_context, jt, dbuilder, isboxed); } static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) @@ -285,30 +287,36 @@ static Value *get_gc_root_for(const jl_cgval_t &x) // --- emitting pointers directly into code --- -static Constant *literal_static_pointer_val(jl_codectx_t &ctx, const void *p, Type *T = T_pjlvalue) -{ - // this function will emit a static pointer into the generated code - // the generated code will only be valid during the current session, - // and thus, this should typically be avoided in new API's -#if defined(_P64) - return ConstantExpr::getIntToPtr(ConstantInt::get(T_int64, (uint64_t)p), T); -#else - return ConstantExpr::getIntToPtr(ConstantInt::get(T_int32, (uint32_t)p), T); -#endif -} +static inline Constant *literal_static_pointer_val(const void *p, Type *T = T_pjlvalue); static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) { - // emit a GlobalVariable for a jl_value_t named "cname" - return jl_get_global_for(cname, addr, jl_Module); + // first see if there already is a GlobalVariable for this address + GlobalVariable* &gv = ctx.global_targets[addr]; + Module *M = jl_Module; + if (!gv) { + // otherwise emit a new GlobalVariable for a jl_value_t named "cname" + std::stringstream gvname; + gvname << cname << globalUnique++; + // no existing GlobalVariable, create one and store it + gv = new GlobalVariable(*M, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); + } + else if (gv->getParent() != M) { + // re-use the same name, but move it to the new module + // this will help simplify merging them later + gv = prepare_global_in(M, gv); + } + return gv; } static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) { // emit a GlobalVariable for a jl_value_t, using the prefix, name, and module to // to create a readable name of the form prefixModA.ModB.name - size_t len = strlen(jl_symbol_name(name))+strlen(prefix)+1; + size_t len = strlen(jl_symbol_name(name)) + strlen(prefix) + 1; jl_module_t *parent = mod, *prev = NULL; while (parent != NULL && parent != prev) { len += strlen(jl_symbol_name(parent->name))+1; @@ -317,15 +325,14 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, j } char *fullname = (char*)alloca(len); strcpy(fullname, prefix); - int skipfirst = jl_symbol_name(name)[0] == '@'; - len -= strlen(jl_symbol_name(name)) + 1 - skipfirst; - strcpy(fullname + len, jl_symbol_name(name) + skipfirst); + len -= strlen(jl_symbol_name(name)) + 1; + strcpy(fullname + len, jl_symbol_name(name)); parent = mod; prev = NULL; while (parent != NULL && parent != prev) { - size_t part = strlen(jl_symbol_name(parent->name))+1-skipfirst; - strcpy(fullname+len-part,jl_symbol_name(parent->name)+skipfirst); - fullname[len-1] = '.'; + size_t part = strlen(jl_symbol_name(parent->name)) + 1; + strcpy(fullname + len - part, jl_symbol_name(parent->name)); + fullname[len - 1] = '.'; len -= part; prev = parent; parent = parent->parent; @@ -342,7 +349,7 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) Module *M = jl_Module; GlobalVariable *gv = new GlobalVariable( *M, T_pjlvalue, true, GlobalVariable::PrivateLinkage, - literal_static_pointer_val(ctx, p)); + literal_static_pointer_val(p)); gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); return gv; } @@ -449,7 +456,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) if (p == NULL) return V_null; if (!imaging_mode) - return literal_static_pointer_val(ctx, p); + return literal_static_pointer_val(p); Value *pgv = literal_pointer_val_slot(ctx, p); return tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable( ctx.builder.CreateLoad(T_pjlvalue, pgv), false, jl_typeof(p))); @@ -461,7 +468,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p) if (p == NULL) return V_null; if (!imaging_mode) - return literal_static_pointer_val(ctx, p); + return literal_static_pointer_val(p); // bindings are prefixed with jl_bnd# Value *pgv = julia_pgv(ctx, "jl_bnd#", p->name, p->owner, p); return tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable( @@ -508,14 +515,12 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b) ctx.builder.CreateLoad(T_pjlvalue, julia_pgv(ctx, "*", b->name, b->owner, b))), T_pprjlvalue); else - bv = ConstantExpr::getBitCast(literal_static_pointer_val(ctx, b), T_pprjlvalue); + bv = ConstantExpr::getBitCast(literal_static_pointer_val(b), T_pprjlvalue); return julia_binding_gv(ctx, bv); } // --- mapping between julia and llvm types --- -static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua_env, bool *isboxed, bool llvmcall=false); - static unsigned convert_struct_offset(Type *lty, unsigned byte_offset) { const DataLayout &DL = jl_data_layout; @@ -536,8 +541,9 @@ static Value *emit_struct_gep(jl_codectx_t &ctx, Type *lty, Value *base, unsigne return ctx.builder.CreateConstInBoundsGEP2_32(lty, base, 0, idx); } -extern "C" { -JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed) +static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, jl_value_t *jt, jl_unionall_t *ua, bool *isboxed, bool llvmcall=false); + +static Type *_julia_type_to_llvm(jl_codegen_params_t *ctx, jl_value_t *jt, bool *isboxed) { // this function converts a Julia Type into the equivalent LLVM type if (isboxed) *isboxed = false; @@ -546,15 +552,26 @@ JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed) if (jl_is_concrete_immutable(jt)) { if (jl_datatype_nbits(jt) == 0) return T_void; - Type *t = julia_struct_to_llvm(jt, NULL, isboxed); + Type *t = _julia_struct_to_llvm(ctx, jt, NULL, isboxed); assert(t != NULL); return t; } if (isboxed) *isboxed = true; return T_prjlvalue; } + +static Type *julia_type_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, bool *isboxed) +{ + return _julia_type_to_llvm(&ctx.emission_context, jt, isboxed); +} + +extern "C" JL_DLLEXPORT +Type *jl_type_to_llvm(jl_value_t *jt, bool *isboxed) +{ + return _julia_type_to_llvm(NULL, jt, isboxed); } + // converts a julia bitstype into the equivalent LLVM bitstype static Type *bitstype_to_llvm(jl_value_t *bt, bool llvmcall = false) { @@ -598,7 +615,7 @@ static bool jl_type_hasptr(jl_value_t* typ) // fields depend on any of the parameters of the containing type) static bool julia_struct_has_layout(jl_datatype_t *dt, jl_unionall_t *ua) { - if (dt->layout || dt->struct_decl) + if (dt->layout) return true; if (ua) { jl_svec_t *types = jl_get_fieldtypes(dt); @@ -620,7 +637,7 @@ static unsigned jl_field_align(jl_datatype_t *dt, size_t i) return std::min({al, (unsigned)jl_datatype_align(dt), (unsigned)JL_HEAP_ALIGNMENT}); } -static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isboxed, bool llvmcall) +static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, jl_value_t *jt, jl_unionall_t *ua_env, bool *isboxed, bool llvmcall) { // this function converts a Julia Type into the equivalent LLVM struct // use this where C-compatible (unboxed) structs are desired @@ -630,111 +647,109 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox return T_void; if (jl_is_primitivetype(jt)) return bitstype_to_llvm(jt, llvmcall); - if (jl_is_structtype(jt)) { - jl_datatype_t *jst = (jl_datatype_t*)jt; + jl_datatype_t *jst = (jl_datatype_t*)jt; + if (jl_is_structtype(jt) && !(jst->layout && jl_is_layout_opaque(jst->layout))) { bool isTuple = jl_is_tuple_type(jt); - // don't use pre-filled struct_decl for llvmcall - if (!llvmcall && (jst->struct_decl != NULL)) - return (Type*)jst->struct_decl; - if (jl_is_structtype(jt) && !(jst->layout && jl_is_layout_opaque(jst->layout))) { - jl_svec_t *ftypes = jl_get_fieldtypes(jst); - size_t i, ntypes = jl_svec_len(ftypes); - if (ntypes == 0 || (jst->layout && jl_datatype_nbits(jst) == 0)) - return T_void; - if (!julia_struct_has_layout(jst, ua)) - return NULL; - std::vector latypes(0); - bool isarray = true; - bool isvector = true; - jl_value_t *jlasttype = NULL; - Type *lasttype = NULL; - bool allghost = true; - for (i = 0; i < ntypes; i++) { - jl_value_t *ty = jl_svecref(ftypes, i); - if (jlasttype != NULL && ty != jlasttype) - isvector = false; - jlasttype = ty; - size_t fsz = 0, al = 0; - bool isptr = !jl_islayout_inline(ty, &fsz, &al); - if (jst->layout) { - assert(isptr == jl_field_isptr(jst, i)); - assert((isptr ? sizeof(void*) : fsz + jl_is_uniontype(ty)) == jl_field_size(jst, i)); - } - Type *lty; - if (isptr) { - lty = T_prjlvalue; - isvector = false; - } - else if (ty == (jl_value_t*)jl_bool_type) { - lty = T_int8; - } - else if (jl_is_uniontype(ty)) { - // pick an Integer type size such that alignment will generally be correct, - // and always end with an Int8 (selector byte). - // We may need to insert padding first to get to the right offset - if (al > MAX_ALIGN) { - Type *AlignmentType = ArrayType::get(VectorType::get(T_int8, al), 0); - latypes.push_back(AlignmentType); - al = MAX_ALIGN; - } - assert(al <= jl_field_align(jst, i)); - Type *AlignmentType = IntegerType::get(jl_LLVMContext, 8 * al); - unsigned NumATy = fsz / al; - unsigned remainder = fsz % al; - while (NumATy--) - latypes.push_back(AlignmentType); - while (remainder--) - latypes.push_back(T_int8); - latypes.push_back(T_int8); - isarray = false; - allghost = false; - continue; - } - else { - lty = julia_struct_to_llvm(ty, NULL, &isptr, llvmcall); - assert(!isptr); - } - if (lasttype != NULL && lasttype != lty) - isarray = false; - lasttype = lty; - if (!type_is_ghost(lty)) { - allghost = false; - latypes.push_back(lty); - } + jl_svec_t *ftypes = jl_get_fieldtypes(jst); + size_t i, ntypes = jl_svec_len(ftypes); + if (ntypes == 0 || (jst->layout && jl_datatype_nbits(jst) == 0)) + return T_void; + Type *_struct_decl = NULL; + // TODO: we should probably make a temporary root for `jst` somewhere + // don't use pre-filled struct_decl for llvmcall (f16, etc. may be different) + Type *&struct_decl = (ctx && !llvmcall ? ctx->llvmtypes[jst] : _struct_decl); + if (struct_decl) + return struct_decl; + if (!julia_struct_has_layout(jst, ua_env)) + return NULL; + std::vector latypes(0); + bool isarray = true; + bool isvector = true; + jl_value_t *jlasttype = NULL; + Type *lasttype = NULL; + bool allghost = true; + for (i = 0; i < ntypes; i++) { + jl_value_t *ty = jl_svecref(ftypes, i); + if (jlasttype != NULL && ty != jlasttype) + isvector = false; + jlasttype = ty; + size_t fsz = 0, al = 0; + bool isptr = !jl_islayout_inline(ty, &fsz, &al); + if (jst->layout) { + assert(isptr == jl_field_isptr(jst, i)); + assert((isptr ? sizeof(void*) : fsz + jl_is_uniontype(ty)) == jl_field_size(jst, i)); } - Type *decl; - if (allghost) { - assert(jst->layout == NULL); // otherwise should have been caught above - decl = T_void; + Type *lty; + if (isptr) { + lty = T_prjlvalue; + isvector = false; } - else if (jl_is_vecelement_type(jt) && !jl_is_uniontype(jl_svecref(ftypes, 0))) { - // VecElement type is unwrapped in LLVM (when possible) - decl = latypes[0]; + else if (ty == (jl_value_t*)jl_bool_type) { + lty = T_int8; } - else if (isarray && !type_is_ghost(lasttype)) { - if (isTuple && isvector && jl_special_vector_alignment(ntypes, jlasttype) != 0) - decl = VectorType::get(lasttype, ntypes); - else - decl = ArrayType::get(lasttype, ntypes); + else if (jl_is_uniontype(ty)) { + // pick an Integer type size such that alignment will generally be correct, + // and always end with an Int8 (selector byte). + // We may need to insert padding first to get to the right offset + if (al > MAX_ALIGN) { + Type *AlignmentType = ArrayType::get(VectorType::get(T_int8, al), 0); + latypes.push_back(AlignmentType); + al = MAX_ALIGN; + } + assert(al <= jl_field_align(jst, i)); + Type *AlignmentType = IntegerType::get(jl_LLVMContext, 8 * al); + unsigned NumATy = fsz / al; + unsigned remainder = fsz % al; + assert(NumATy > 0); + while (NumATy--) + latypes.push_back(AlignmentType); + while (remainder--) + latypes.push_back(T_int8); + latypes.push_back(T_int8); + isarray = false; + allghost = false; + continue; } else { + lty = _julia_struct_to_llvm(ctx, ty, NULL, &isptr, llvmcall); + assert(!isptr); + } + if (lasttype != NULL && lasttype != lty) + isarray = false; + lasttype = lty; + if (!type_is_ghost(lty)) { + allghost = false; + latypes.push_back(lty); + } + } + if (allghost) { + assert(jst->layout == NULL); // otherwise should have been caught above + struct_decl = T_void; + } + else if (jl_is_vecelement_type(jt) && !jl_is_uniontype(jl_svecref(ftypes, 0))) { + // VecElement type is unwrapped in LLVM (when possible) + struct_decl = latypes[0]; + } + else if (isarray && !type_is_ghost(lasttype)) { + if (isTuple && isvector && jl_special_vector_alignment(ntypes, jlasttype) != 0) + struct_decl = VectorType::get(lasttype, ntypes); + else + struct_decl = ArrayType::get(lasttype, ntypes); + } + else { #if 0 // stress-test code that tries to assume julia-index == llvm-index - // (also requires change to emit_new_struct to not assume 0 == 0) - if (!isTuple && latypes.size() > 1) { - Type *NoopType = ArrayType::get(T_int1, 0); - latypes.insert(latypes.begin(), NoopType); - } -#endif - decl = StructType::get(jl_LLVMContext, latypes); + // (also requires change to emit_new_struct to not assume 0 == 0) + if (!isTuple && latypes.size() > 1) { + Type *NoopType = ArrayType::get(T_int1, 0); + latypes.insert(latypes.begin(), NoopType); } - // don't use struct_decl cache for llvmcall - if (!llvmcall) - jst->struct_decl = decl; - return decl; +#endif + struct_decl = StructType::get(jl_LLVMContext, latypes); } + return struct_decl; } - // TODO: enable this (with tests): - // if (jl_is_uniontype(jt)) { + // TODO: enable this (with tests) to change ccall calling convention for Union: + // if (jl_is_uniontype(ty)) { // // pick an Integer type size such that alignment will be correct // // and always end with an Int8 (selector byte) // lty = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), fsz / al); @@ -750,6 +765,11 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox return T_prjlvalue; } +static Type *julia_struct_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, jl_unionall_t *ua, bool *isboxed) +{ + return _julia_struct_to_llvm(&ctx.emission_context, jt, ua, isboxed); +} + static bool is_datatype_all_pointers(jl_datatype_t *dt) { size_t i, l = jl_datatype_nfields(dt); @@ -1347,7 +1367,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j bool maybe_null_if_boxed = true, unsigned alignment = 0) { bool isboxed; - Type *elty = julia_type_to_llvm(jltype, &isboxed); + Type *elty = julia_type_to_llvm(ctx, jltype, &isboxed); if (type_is_ghost(elty)) return ghostValue(jltype); Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); @@ -1396,7 +1416,7 @@ static void typed_store(jl_codectx_t &ctx, unsigned alignment = 0) { bool isboxed; - Type *elty = julia_type_to_llvm(jltype, &isboxed); + Type *elty = julia_type_to_llvm(ctx, jltype, &isboxed); if (type_is_ghost(elty)) return; Value *r; @@ -1432,13 +1452,14 @@ static Value *julia_bool(jl_codectx_t &ctx, Value *cond) // --- accessing the representations of built-in data types --- -static Constant *julia_const_to_llvm(jl_value_t *e); +static Constant *julia_const_to_llvm(jl_codectx_t &ctx, jl_value_t *e); + static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x) { assert(x.ispointer()); Value *data = x.V; if (x.constant) { - Constant *val = julia_const_to_llvm(x.constant); + Constant *val = julia_const_to_llvm(ctx, x.constant); if (val) { data = get_pointer_to_constant(val, "", *jl_Module); } @@ -1637,7 +1658,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, Value *ptr = maybe_decay_tracked(data_pointer(ctx, strct)); if (!stt->mutabl && !(maybe_null && jft == (jl_value_t*)jl_bool_type)) { // just compute the pointer and let user load it when necessary - Type *fty = julia_type_to_llvm(jft); + Type *fty = julia_type_to_llvm(ctx, jft); Value *addr = ctx.builder.CreateInBoundsGEP(fty, emit_bitcast(ctx, ptr, PointerType::get(fty, 0)), idx); *ret = mark_julia_slot(addr, jft, NULL, strct.tbaa); return true; @@ -1662,14 +1683,14 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st raise_exception(ctx, literal_pointer_val(ctx, jl_undefref_exception)); return jl_cgval_t(); // unreachable } - if (type_is_ghost(julia_type_to_llvm(jfty))) + if (type_is_ghost(julia_type_to_llvm(ctx, jfty))) return ghostValue(jfty); bool maybe_null = idx >= (unsigned)jt->ninitialized; size_t byte_offset = jl_field_offset(jt, idx); if (strct.ispointer()) { Value *staddr = maybe_decay_tracked(data_pointer(ctx, strct)); bool isboxed; - Type *lt = julia_type_to_llvm((jl_value_t*)jt, &isboxed); + Type *lt = julia_type_to_llvm(ctx, (jl_value_t*)jt, &isboxed); Value *addr; if (isboxed) { // byte_offset == 0 is an important special case here, e.g. @@ -2249,7 +2270,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t else if (jb == jl_ssavalue_type) { unsigned zero = 0; Value *v = as_value(ctx, t, vinfo); - assert(v->getType() == jl_ssavalue_type->struct_decl); + assert(v->getType() == ctx.emission_context.llvmtypes[jl_ssavalue_type]); v = ctx.builder.CreateExtractValue(v, makeArrayRef(&zero, 1)); box = call_with_attrs(ctx, box_ssavalue_func, v); } @@ -2368,7 +2389,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB [&](unsigned idx, jl_datatype_t *jt) { if (idx < skip.size() && skip[idx]) return; - Type *t = julia_type_to_llvm((jl_value_t*)jt); + Type *t = julia_type_to_llvm(ctx, (jl_value_t*)jt); BasicBlock *tempBB = BasicBlock::Create(jl_LLVMContext, "box_union", ctx.f); ctx.builder.SetInsertPoint(tempBB); switchInst->addCase(ConstantInt::get(T_int8, idx), tempBB); @@ -2437,7 +2458,7 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo) else { assert(vinfo.V && "Missing data for unboxed value."); assert(jl_is_concrete_immutable(jt) && "This type shouldn't have been unboxed."); - Type *t = julia_type_to_llvm(jt); + Type *t = julia_type_to_llvm(ctx, jt); assert(!type_is_ghost(t)); // ghost values should have been handled by vinfo.constant above! box = _boxed_special(ctx, vinfo, t); if (!box) { @@ -2459,7 +2480,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con ctx.builder.CreateAlignedStore(UndefValue::get(ai->getAllocatedType()), ai, ai->getAlignment()); if (jl_is_concrete_type(src.typ) || src.constant) { jl_value_t *typ = src.constant ? jl_typeof(src.constant) : src.typ; - Type *store_ty = julia_type_to_llvm(typ); + Type *store_ty = julia_type_to_llvm(ctx, typ); assert(skip || jl_is_pointerfree(typ)); if (jl_is_pointerfree(typ)) { if (!src.ispointer() || src.constant) { @@ -2664,7 +2685,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg size_t nf = jl_datatype_nfields(sty); if (nf > 0 || sty->mutabl) { if (deserves_stack(ty)) { - Type *lt = julia_type_to_llvm(ty); + Type *lt = julia_type_to_llvm(ctx, ty); unsigned na = nargs < nf ? nargs : nf; // whether we should perform the initialization with the struct as a IR value @@ -2701,7 +2722,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg fval_info = update_julia_type(ctx, fval_info, jtype); if (type_is_ghost(lt)) continue; - Type *fty = julia_type_to_llvm(jtype); + Type *fty = julia_type_to_llvm(ctx, jtype); if (type_is_ghost(fty)) continue; Value *dest = NULL; @@ -2836,7 +2857,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg if (jl_datatype_nbits(sty) == 0) return ghostValue(sty); bool isboxed; - Type *lt = julia_type_to_llvm(ty, &isboxed); + Type *lt = julia_type_to_llvm(ctx, ty, &isboxed); assert(!isboxed); return mark_julia_type(ctx, UndefValue::get(lt), false, ty); } @@ -2867,7 +2888,7 @@ static Value *emit_defer_signal(jl_codectx_t &ctx) static int compare_cgparams(const jl_cgparams_t *a, const jl_cgparams_t *b) { - return (a->cached == b->cached) && + return // language features (a->track_allocations == b->track_allocations) && (a->code_coverage == b->code_coverage) && diff --git a/src/codegen.cpp b/src/codegen.cpp index 7a4d9fba5050f..bdbda520bf63a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -11,6 +11,9 @@ #if defined(_CPU_X86_) #define JL_NEED_FLOATTEMP_VAR 1 #endif +#if defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_) +#define JL_DISABLE_FPO +#endif #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS @@ -75,7 +78,7 @@ #include #include #endif -#include +#include using namespace llvm; @@ -92,11 +95,6 @@ typedef Instruction TerminatorInst; #include "processor.h" #include "julia_assert.h" -// LLVM version compatibility macros -legacy::PassManager *jl_globalPM; - -#define DIFlagZero (DINode::FlagZero) - extern "C" { #include "builtin_proto.h" @@ -143,20 +141,19 @@ extern void _chkstk(void); #define DISABLE_FLOAT16 // llvm state -JL_DLLEXPORT LLVMContext &jl_LLVMContext = *(new LLVMContext()); -TargetMachine *jl_TargetMachine; - extern JITEventListener *CreateJuliaJITEventListener(); // for image reloading bool imaging_mode = false; +// shared llvm state +static LLVMContext &jl_LLVMContext = *(new LLVMContext()); +TargetMachine *jl_TargetMachine; Module *shadow_output; +static DataLayout &jl_data_layout = *(new DataLayout("")); #define jl_Module ctx.f->getParent() #define jl_builderModule(builder) (builder).GetInsertBlock()->getParent()->getParent() -static DataLayout &jl_data_layout = *(new DataLayout("")); - // types static Type *T_jlvalue; static Type *T_pjlvalue; @@ -237,9 +234,6 @@ static DISubroutineType *jl_di_func_null_sig; // constants static Constant *V_null; -extern "C" { -JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed=NULL); -} static bool type_is_ghost(Type *ty) { return (ty == T_void || ty->isEmptyTy()); @@ -314,11 +308,6 @@ static Function *jltypeassert_func; static Function *jlgetnthfieldchecked_func; //static Function *jlsetnthfield_func; static Function *jlgetcfunctiontrampoline_func; -#ifdef _OS_WINDOWS_ -#if defined(_CPU_X86_64_) -Function *juliapersonality_func; -#endif -#endif static Function *diff_gc_total_bytes_func; static Function *sync_gc_total_bytes_func; static Function *jlarray_data_owner_func; @@ -337,7 +326,7 @@ static std::map builtin_func_map; extern "C" { int globalUnique = 0; int jl_default_debug_info_kind = (int) DICompileUnit::DebugEmissionKind::FullDebug; - jl_cgparams_t jl_default_cgparams = {1, 1, 1, 1, 0, + jl_cgparams_t jl_default_cgparams = {1, 1, 1, 0, #ifdef _OS_WINDOWS_ 0, #else @@ -510,28 +499,14 @@ struct jl_varinfo_t { } }; -struct jl_returninfo_t { - Function *decl; - enum CallingConv { - Boxed = 0, - Register, - SRet, - Union, - Ghosts - } cc; - size_t union_bytes; - size_t union_align; - size_t union_minalign; - unsigned return_roots; -}; - -static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype); - // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. class jl_codectx_t { public: IRBuilder<> builder; + jl_codegen_params_t &emission_context; + jl_codegen_call_targets_t &call_targets; + std::map &global_targets; Function *f = NULL; // local var info. globals are not in here. std::vector slots; @@ -541,6 +516,7 @@ class jl_codectx_t { std::vector ssavalue_assigned; jl_module_t *module = NULL; jl_method_instance_t *linfo = NULL; + jl_value_t *rettype = NULL; jl_code_info_t *source = NULL; jl_array_t *code = NULL; size_t world = 0; @@ -563,16 +539,25 @@ class jl_codectx_t { Value *world_age_field = NULL; bool debug_enabled = false; + bool use_cache = false; const jl_cgparams_t *params = NULL; - jl_codectx_t(LLVMContext &llvmctx) - : builder(llvmctx) { } + jl_codectx_t(LLVMContext &llvmctx, jl_codegen_params_t ¶ms) + : builder(llvmctx), + emission_context(params), + call_targets(params.workqueue), + global_targets(params.globals), + world(params.world), + use_cache(params.cache), + params(params.params) { } ~jl_codectx_t() { assert(this->roots == NULL); } }; +static Type *julia_type_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, bool *isboxed = NULL); +static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype); static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval = -1); static Value *global_binding_pointer(jl_codectx_t &ctx, jl_module_t *m, jl_sym_t *s, jl_binding_t **pbnd, bool assign); @@ -653,9 +638,8 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) } else { typ = jl_typeof(jv); - if (type_is_ghost(julia_type_to_llvm(typ))) { + if (jl_is_datatype_singleton((jl_datatype_t*)typ)) return ghostValue(typ); - } } jl_cgval_t constant(NULL, NULL, true, typ, NULL); constant.constant = jv; @@ -718,7 +702,7 @@ static inline jl_cgval_t mark_julia_type(jl_codectx_t &ctx, Value *v, bool isbox return ghostValue(typ); } } - Type *T = julia_type_to_llvm(typ); + Type *T = julia_type_to_llvm(ctx, typ); if (type_is_ghost(T)) { return ghostValue(typ); } @@ -771,7 +755,7 @@ static inline jl_cgval_t update_julia_type(jl_codectx_t &ctx, const jl_cgval_t & if (!jl_is_concrete_type(typ)) return v; // not generally worth trying to change type info (which would require recomputing tindex) } - Type *T = julia_type_to_llvm(typ); + Type *T = julia_type_to_llvm(ctx, typ); if (type_is_ghost(T)) return ghostValue(typ); return jl_cgval_t(v, typ, NULL); @@ -838,23 +822,6 @@ static void CreateConditionalAbort(IRBuilder<> &irbuilder, Value *test) #include "cgutils.cpp" -static void jl_rethrow_with_add(const char *fmt, ...) -{ - if (jl_typeis(jl_current_exception(), jl_errorexception_type)) { - char *str = jl_string_data(jl_fieldref(jl_current_exception(),0)); - char buf[1024]; - va_list args; - va_start(args, fmt); - int nc = vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - nc += snprintf(buf+nc, sizeof(buf)-nc, ": %s", str); - jl_value_t *msg = jl_pchar_to_string(buf, nc); - JL_GC_PUSH1(&msg); - jl_throw(jl_new_struct(jl_errorexception_type, msg)); - } - jl_rethrow(); -} - static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ) { // previous value was a split union, compute new index, or box @@ -1007,7 +974,7 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return ghostValue(typ); // normalize TypeofBottom to Type{Union{}} if (v.typ == typ || v.typ == jl_bottom_type || jl_egal(v.typ, typ)) return v; // fast-path - Type *T = julia_type_to_llvm(typ); + Type *T = julia_type_to_llvm(ctx, typ); if (type_is_ghost(T)) return ghostValue(typ); Value *new_tindex = NULL; @@ -1071,307 +1038,6 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return jl_cgval_t(v, typ, new_tindex); } -// Snooping on which functions are being compiled, and how long it takes -static JL_STREAM *dump_compiles_stream = NULL; -static bool nested_compile = false; -static uint64_t last_time = 0; -extern "C" JL_DLLEXPORT -void jl_dump_compiles(void *s) -{ - dump_compiles_stream = (JL_STREAM*)s; -} - -// --- entry point --- -//static int n_emit=0; -static std::unique_ptr emit_function( - jl_method_instance_t *lam, - jl_code_info_t *src, - jl_value_t *jlrettype, - size_t world, - jl_llvm_functions_t *declarations, - const jl_cgparams_t *params); -void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL); - -const char *name_from_method_instance(jl_method_instance_t *mi) -{ - return jl_is_method(mi->def.method) ? jl_symbol_name(mi->def.method->name) : "top-level scope"; -} - -// this generates llvm code for the lambda info -// and adds the result to the jitlayers -// (and the shadow module), but doesn't yet compile -// or generate object code for it -extern "C" -jl_code_instance_t *jl_compile_linfo(jl_method_instance_t *mi, jl_code_info_t *src, size_t world, const jl_cgparams_t *params) -{ - // TODO: Merge with jl_dump_compiles? - static ios_t f_precompile; - static JL_STREAM* s_precompile = NULL; - - // N.B.: `src` may have not been rooted by the caller. - JL_TIMING(CODEGEN); - assert(jl_is_method_instance(mi)); - jl_code_instance_t *codeinst = NULL; - - if (params != &jl_default_cgparams /* fast path */ && - !compare_cgparams(params, &jl_default_cgparams) && params->cached) - jl_error("functions compiled with custom codegen params mustn't be cached"); - - // Fast path for the already-compiled case - if (jl_is_method(mi->def.method)) { - codeinst = mi->cache; - while (codeinst) { - if (codeinst->min_world <= world && world <= codeinst->max_world) { - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - bool already_compiled = params->cached && decls.functionObject != NULL; - if (!src) { - if (already_compiled || codeinst->invoke == jl_fptr_const_return) - return codeinst; - } - else if (already_compiled) { - return codeinst; - } - } - codeinst = codeinst->next; - } - } - - JL_GC_PUSH2(&src, &codeinst); - JL_LOCK(&codegen_lock); - - // Codegen lock held in this block - { - // Step 1: Re-check if this was already compiled (it may have been while - // we waited at the lock). - if (!jl_is_method(mi->def.method)) { - src = (jl_code_info_t*)mi->uninferred; - if (!src || !jl_is_code_info(src)) - goto locked_out; - codeinst = mi->cache; - while (codeinst) { - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - if (decls.functionObject != NULL || codeinst->invoke == jl_fptr_const_return) { - goto locked_out; - } - codeinst = codeinst->next; - } - } - else if (!src) { - // If the caller didn't provide the source, - // try to infer it for ourself, but first, repeat the search for it already compiled - if (jl_is_method(mi->def.method)) { - codeinst = mi->cache; - while (codeinst) { - if (codeinst->min_world <= world && world <= codeinst->max_world) { - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - bool already_compiled = params->cached && decls.functionObject != NULL; - if (already_compiled) { - goto locked_out; - } - } - codeinst = codeinst->next; - } - } - - // see if it is inferred - codeinst = mi->cache; - while (codeinst) { - src = (jl_code_info_t*)codeinst->inferred; - if (src && codeinst->min_world <= world && world <= codeinst->max_world) { - break; - } - codeinst = codeinst->next; - } - if (!codeinst) { - // declare a failure to compile - goto locked_out; - } - // found inferred code, prep it for codegen - if ((jl_value_t*)src != jl_nothing) - src = jl_uncompress_ast(mi->def.method, codeinst, (jl_array_t*)src); - if (!jl_is_code_info(src)) { - src = jl_type_infer(mi, world, 0); - if (!src) - goto locked_out; - codeinst = jl_get_method_inferred(mi, src->rettype, src->min_world, src->max_world); - if (!codeinst->inferred) - codeinst->inferred = jl_nothing; - } - // check again for recursion - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - bool already_compiled = params->cached && decls.functionObject != NULL; - if (already_compiled || codeinst->invoke == jl_fptr_const_return) - goto locked_out; - } - else { - if (params->cached) { - codeinst = mi->cache; - while (codeinst) { - if (codeinst->min_world <= world && world <= codeinst->max_world) { - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - bool already_compiled = params->cached && decls.functionObject != NULL; - if (already_compiled) { - // similar to above, but never returns a NULL - // decl (unless compile fails), even if invoke == jl_fptr_const_return - goto locked_out; - } - } - codeinst = codeinst->next; - } - } - assert((jl_value_t*)src != jl_nothing); - src = jl_uncompress_ast(mi->def.method, NULL, (jl_array_t*)src); - } - assert(jl_is_code_info(src)); - - if (!codeinst) { - codeinst = jl_get_method_inferred(mi, src->rettype, src->min_world, src->max_world); - if (src->inferred && !codeinst->inferred) - codeinst->inferred = jl_nothing; - } - if (!params->cached) { - // caller demanded that we make a new copy - jl_ptls_t ptls = jl_get_ptls_states(); - jl_code_instance_t *uncached = (jl_code_instance_t*)jl_gc_alloc(ptls, sizeof(jl_code_instance_t), - jl_code_instance_type); - *uncached = *codeinst; - uncached->functionObjectsDecls.functionObject = NULL; - uncached->functionObjectsDecls.specFunctionObject = NULL; - uncached->inferred = jl_nothing; - if (uncached->invoke != jl_fptr_const_return) - uncached->invoke = NULL; - uncached->specptr.fptr = NULL; - codeinst = uncached; - } - - // Step 2: setup global state - bool last_n_c = nested_compile; - if (!nested_compile && dump_compiles_stream != NULL) - last_time = jl_hrtime(); - nested_compile = true; - - // Step 3. actually do the work of emitting the function - std::unique_ptr m; - JL_TRY { - m = emit_function(mi, src, codeinst->rettype, world, &codeinst->functionObjectsDecls, params); - //n_emit++; - } - JL_CATCH { - // something failed! this is very bad, since other WIP may be pointing to this function - // but there's not much we can do now. try to clear much of the WIP anyways. - codeinst->functionObjectsDecls.functionObject = NULL; - codeinst->functionObjectsDecls.specFunctionObject = NULL; - nested_compile = last_n_c; - JL_UNLOCK(&codegen_lock); // Might GC - const char *mname = name_from_method_instance(mi); - jl_rethrow_with_add("error compiling %s", mname); - } - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - const char *f = decls.functionObject; - const char *specf = decls.specFunctionObject; - - if (JL_HOOK_TEST(params, module_activation)) { - JL_HOOK_CALL(params, module_activation, 1, jl_box_voidpointer(wrap(m.release()))); - } - else { - // Step 4. Prepare debug info to receive this function - // record that this function name came from this linfo, - // so we can build a reverse mapping for debug-info. - bool toplevel = !jl_is_method(mi->def.method); - if (!toplevel) { - const DataLayout &DL = m->getDataLayout(); - // but don't remember toplevel thunks because - // they may not be rooted in the gc for the life of the program, - // and the runtime doesn't notify us when the code becomes unreachable :( - if (specf) - jl_add_code_in_flight(specf, codeinst, DL); - if (strcmp(f, "jl_fptr_args") && strcmp(f, "jl_fptr_sparam")) - jl_add_code_in_flight(f, codeinst, DL); - } - - // Step 5. Add the result to the execution engine now - jl_finalize_module(m.release(), !toplevel); - } - - if (// don't alter `inferred` when the code is not directly being used - world && - // don't change inferred state - codeinst->inferred) { - if (// keep code when keeping everything - !(JL_DELETE_NON_INLINEABLE) || - // aggressively keep code when debugging level >= 2 - jl_options.debug_level > 1) { - // update the stored code - if (codeinst->inferred != (jl_value_t*)src) { - if (jl_is_method(mi->def.method)) - src = (jl_code_info_t*)jl_compress_ast(mi->def.method, src); - codeinst->inferred = (jl_value_t*)src; - jl_gc_wb(codeinst, src); - } - } - else if (// don't delete toplevel code - jl_is_method(mi->def.method) && - // and there is something to delete (test this before calling jl_ast_flag_inlineable) - codeinst->inferred != jl_nothing && - // don't delete inlineable code, unless it is constant - (codeinst->invoke == jl_fptr_const_return || !jl_ast_flag_inlineable((jl_array_t*)codeinst->inferred)) && - // don't delete code when generating a precompile file - !imaging_mode) { - // if not inlineable, code won't be needed again - codeinst->inferred = jl_nothing; - } - } - - // Step 6: Done compiling: Restore global state - nested_compile = last_n_c; - } - - JL_UNLOCK(&codegen_lock); // Might GC - - // If logging of the compilation stream is enabled then dump the function to the stream - // ... unless mi->def isn't defined here meaning the function is a toplevel thunk and - // would have its CodeInfo printed in the stream, which might contain double-quotes that - // would not be properly escaped given the double-quotes added to the stream below. - if (jl_is_method(mi->def.method)) { - if (jl_options.trace_compile != NULL) { - if (s_precompile == NULL) { - const char* t = jl_options.trace_compile; - if (!strncmp(t, "stderr", 6)) - s_precompile = JL_STDERR; - else { - if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL) - jl_errorf("cannot open precompile statement file \"%s\" for writing", t); - s_precompile = (JL_STREAM*) &f_precompile; - } - } - if (!jl_has_free_typevars(mi->specTypes)) { - jl_printf(s_precompile, "precompile("); - jl_static_show(s_precompile, mi->specTypes); - jl_printf(s_precompile, ")\n"); - - if (s_precompile != JL_STDERR) - ios_flush(&f_precompile); - } - } - if (dump_compiles_stream != NULL) { - uint64_t this_time = jl_hrtime(); - jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", this_time - last_time); - jl_static_show(dump_compiles_stream, mi->specTypes); - jl_printf(dump_compiles_stream, "\"\n"); - last_time = this_time; - } - } - JL_GC_POP(); - return codeinst; - -locked_out: - JL_UNLOCK(&codegen_lock); - JL_GC_POP(); - return codeinst; -} - -#define getModuleFlag(m,str) m->getModuleFlag(str) - static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default_cgparams) { if (JL_HOOK_TEST(params, module_setup)) { @@ -1382,7 +1048,7 @@ static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default // Some linkers (*cough* OS X) don't understand DWARF v4, so we use v2 in // imaging mode. The structure of v4 is slightly nicer for debugging JIT // code. - if (!getModuleFlag(m,"Dwarf Version")) { + if (!m->getModuleFlag("Dwarf Version")) { int dwarf_version = 4; #ifdef _OS_DARWIN_ if (imaging_mode) @@ -1390,359 +1056,93 @@ static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default #endif m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", dwarf_version); } - if (!getModuleFlag(m,"Debug Info Version")) + if (!m->getModuleFlag("Debug Info Version")) m->addModuleFlag(llvm::Module::Error, "Debug Info Version", llvm::DEBUG_METADATA_VERSION); m->setDataLayout(jl_data_layout); m->setTargetTriple(jl_TargetMachine->getTargetTriple().str()); - } -// this ensures that llvmf has been emitted to the execution engine, -// returning the function pointer to it -static uint64_t getAddressForFunction(StringRef fname) +static void jl_init_function(Function *F) { - JL_TIMING(LLVM_EMIT); - if (fname == "jl_fptr_args") - return (uintptr_t)&jl_fptr_args; - else if (fname == "jl_fptr_sparam") - return (uintptr_t)&jl_fptr_sparam; -#ifdef JL_DEBUG_BUILD - llvm::raw_fd_ostream out(1, false); + // set any attributes that *must* be set on all functions +#if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) + // tell Win32 to realign the stack to the next 16-byte boundary + // upon entry to any function. This achieves compatibility + // with both MinGW-GCC (which assumes an 16-byte-aligned stack) and + // i686 Windows (which uses a 4-byte-aligned stack) + AttrBuilder attr; + attr.addStackAlignmentAttr(16); + F->addAttributes(AttributeList::FunctionIndex, attr); +#endif +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) + F->setHasUWTable(); // force NeedsWinEH +#endif +#ifdef JL_DISABLE_FPO +#if LLVM_VERSION_MAJOR >= 8 + F->addFnAttr("frame-pointer", "all"); +#else + F->addFnAttr("no-frame-pointer-elim", "true"); #endif - jl_finalize_function(fname); - uint64_t ret = jl_ExecutionEngine->getFunctionAddress(fname); - return ret; -} - -// convenience helper exported for usage from gdb -extern "C" JL_DLLEXPORT -uint64_t jl_get_llvm_fptr(void *function) -{ - Function *F = (Function*)function; - uint64_t addr = getAddressForFunction(F->getName()); - if (!addr) { - if (auto exp_addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress()) { - addr = exp_addr.get(); - } - } - return addr; -} - -// this finishes compiling a native-code object and sets `output->invoke` -extern "C" -void jl_generate_fptr(jl_code_instance_t *output) -{ - jl_code_instance_t *codeinst = output; - jl_callptr_t fptr; - fptr = codeinst->invoke; - if (fptr != NULL) - return; - - JL_LOCK(&codegen_lock); - fptr = codeinst->invoke; - if (fptr != NULL) { - JL_UNLOCK(&codegen_lock); - return; - } - jl_method_instance_t *unspec = NULL; - if (jl_is_method(codeinst->def->def.method)) { - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - const char *F = decls.functionObject; - const char *specF = decls.specFunctionObject; - if (!F || !jl_can_finalize_function(F) || (specF && !jl_can_finalize_function(specF))) { - // can't compile F in the JIT right now, - // so instead compile an unspecialized version - // and return its fptr instead - if (codeinst->def->def.method->unspecialized) { - unspec = codeinst->def->def.method->unspecialized; - } - if (!unspec) - unspec = jl_get_unspecialized(codeinst->def); // get-or-create the unspecialized version to cache the result - jl_code_info_t *src = (jl_code_info_t*)unspec->def.method->source; - if (src == NULL) { - assert(unspec->def.method->generator); - src = jl_code_for_staged(unspec); - } - jl_code_instance_t *ucache = unspec->cache; - while (ucache) { - if (ucache->min_world <= output->min_world && output->max_world <= ucache->max_world && ucache->invoke != NULL) - break; - ucache = ucache->next; - } - if (codeinst->invoke) { - JL_UNLOCK(&codegen_lock); - return; - } - if (ucache != NULL) { - codeinst->specptr = ucache->specptr; - codeinst->rettype_const = ucache->rettype_const; - if (codeinst->rettype_const) - jl_gc_wb(codeinst, codeinst->rettype_const); - jl_atomic_store_release(&codeinst->invoke, ucache->invoke); - JL_UNLOCK(&codegen_lock); - return; - } - assert(src); - ucache = jl_compile_linfo(unspec, src, codeinst->def->def.method->primary_world, &jl_default_cgparams); - // if we hit a generated function, this might explode now, because - // (unlike normal functions), any call to a generated function - // in the compiler (Core or Core.Compiler) might cause us to die at any time - if (ucache == NULL) - jl_error("Failed to handle @generated function in Core.Compiler module."); - codeinst = ucache; - } - } - - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - const char *F = decls.functionObject; - const char *specF = decls.specFunctionObject; - assert(F && jl_can_finalize_function(F)); - assert(specF && jl_can_finalize_function(specF)); - if (!strcmp(F, "jl_fptr_args")) - fptr = &jl_fptr_args; - else if (!strcmp(F, "jl_fptr_sparam")) - fptr = &jl_fptr_sparam; - else - fptr = (jl_callptr_t)(uintptr_t)getAddressForFunction(F); - assert(fptr != NULL); - void *specptr = (void*)(uintptr_t)getAddressForFunction(specF); - assert(specptr != NULL); - // the fptr should be cached somewhere also - if (codeinst->invoke == NULL) { - jl_atomic_store_release(&codeinst->specptr.fptr, specptr); - jl_atomic_store_release(&codeinst->invoke, fptr); - } - if (codeinst != output && output->invoke == NULL) { - output->specptr = codeinst->specptr; - output->rettype_const = codeinst->rettype_const; - if (output->rettype_const) - jl_gc_wb(output, output->rettype_const); - jl_atomic_store_release(&output->invoke, fptr); - } - JL_UNLOCK(&codegen_lock); // Might GC -} - -static Function *jl_cfunction_object(jl_value_t *f, jl_value_t *declrt, jl_tupletype_t *argt); - -// get the address of a C-callable entry point for a function -extern "C" JL_DLLEXPORT -void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) -{ - JL_LOCK(&codegen_lock); - Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - void *ptr = (void*)getAddressForFunction(llvmf->getName()); - JL_UNLOCK(&codegen_lock); - return ptr; -} - - -// convenience function for debugging from gdb (pre-OrcJIT) -// it generally helps to have define KEEP_BODIES if you plan on using this -extern "C" JL_DLLEXPORT -void *jl_function_ptr_by_llvm_name(char *name) { -#ifdef JL_MSAN_ENABLED - __msan_unpoison_string(name); #endif - return (void*)jl_ExecutionEngine->FindFunctionNamed(name); // returns an llvm::Function* -} - -// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name -extern "C" JL_DLLEXPORT -void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) -{ - JL_LOCK(&codegen_lock); - Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) - uint64_t Addr = getAddressForFunction(llvmf->getName()); - - if (imaging_mode) - llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); - - // make the alias to the shadow_module - GlobalAlias *GA = - GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, name, llvmf, shadow_output); - - // make sure the alias name is valid for the current session - jl_ExecutionEngine->addGlobalMapping(GA, (void*)(uintptr_t)Addr); - JL_UNLOCK(&codegen_lock); } -// --- native code info, and dump function to IR and ASM --- -// Get pointer to llvm::Function instance, compiling if necessary -// for use in reflection from Julia. -// this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: -// misuse will leak memory or cause read-after-free -extern "C" JL_DLLEXPORT -void *jl_get_llvmf_defn(jl_method_instance_t *mi, size_t world, bool getwrapper, bool optimize, const jl_cgparams_t params) +static std::pair uses_specsig(jl_method_instance_t *lam, jl_value_t *rettype, bool prefer_specsig) { - if (jl_is_method(mi->def.method) && mi->def.method->source == NULL && - mi->def.method->generator == NULL) { - // not a generic function - return NULL; - } - - jl_value_t *jlrettype = (jl_value_t*)jl_any_type; - jl_code_info_t *src = NULL; - JL_GC_PUSH2(&src, &jlrettype); - jl_value_t *ci = jl_rettype_inferred(mi, world, world); - if (ci != jl_nothing) { - jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; - src = (jl_code_info_t*)codeinst->inferred; - if ((jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method)) - src = jl_uncompress_ast(mi->def.method, codeinst, (jl_array_t*)src); - jlrettype = codeinst->rettype; + size_t nreq = jl_is_method(lam->def.method) ? lam->def.method->nargs : 0; + int va = 0; + if (nreq > 0 && lam->def.method->isva) { + nreq--; + va = 1; } - if (!src || (jl_value_t*)src == jl_nothing) { - src = jl_type_infer(mi, world, 0); - if (src) - jlrettype = src->rettype; - if (!src && jl_is_method(mi->def.method)) { - src = mi->def.method->generator ? jl_code_for_staged(mi) : (jl_code_info_t*)mi->def.method->source; - if (src && !jl_is_code_info(src) && jl_is_method(mi->def.method)) - src = jl_uncompress_ast(mi->def.method, NULL, (jl_array_t*)src); + jl_value_t *sig = lam->specTypes; + bool needsparams = false; + if (jl_is_method(lam->def.method)) { + if ((size_t)jl_subtype_env_size(lam->def.method->sig) != jl_svec_len(lam->sparam_vals)) + needsparams = true; + for (size_t i = 0; i < jl_svec_len(lam->sparam_vals); ++i) { + if (jl_is_typevar(jl_svecref(lam->sparam_vals, i))) + needsparams = true; } } - if (src == NULL || (jl_value_t*)src == jl_nothing || !jl_is_code_info(src)) - jl_error("source not available for function"); - - // Backup the info for the nested compile - JL_LOCK(&codegen_lock); - - // emit this function into a new module - jl_llvm_functions_t declarations; - std::unique_ptr m; - JL_TRY { - m = emit_function(mi, src, jlrettype, world, &declarations, ¶ms); - } - JL_CATCH { - // something failed! - m.reset(); - JL_UNLOCK(&codegen_lock); // Might GC - const char *mname = name_from_method_instance(mi); - jl_rethrow_with_add("error compiling %s", mname); - } - - if (optimize) - jl_globalPM->run(*m.get()); - - // swap declarations for definitions and destroy declarations - const char *fname = declarations.functionObject; - const char *specfname = declarations.specFunctionObject; - Function *f = NULL; - Function *specf = NULL; - if (specfname) { - specf = cast(m->getNamedValue(specfname)); - free(const_cast(specfname)); - } - f = cast_or_null(m->getNamedValue(fname)); - if (f) // don't try to free sentinel names like "jl_fptr_args" and "jl_fptr_sparam" - free(const_cast(fname)); - assert(specf || f); - // try to clone the name from some linfo, if it exists - // to give the user a (false) sense of stability - jl_code_instance_t *codeinst = mi->cache; - while (codeinst) { - specfname = codeinst->functionObjectsDecls.specFunctionObject; - if (specfname && specf) { - specf->setName(specfname); - } - fname = codeinst->functionObjectsDecls.functionObject; - if (fname && f) { - if (strcmp(fname, "jl_fptr_args") && strcmp(fname, "jl_fptr_sparam")) - f->setName(fname); - break; - } - codeinst = codeinst->next; + if (needsparams) + return std::make_pair(false, true); + if (sig == (jl_value_t*)jl_anytuple_type) + return std::make_pair(false, false); + if (!jl_is_datatype(sig)) + return std::make_pair(false, false); + if (jl_nparams(sig) == 0) + return std::make_pair(false, false); + if (va) { + if (jl_is_vararg_type(jl_tparam(sig, jl_nparams(sig) - 1))) + return std::make_pair(false, false); } - m.release(); // the return object `llvmf` will be the owning pointer - JL_UNLOCK(&codegen_lock); // Might GC - JL_GC_POP(); - if ((getwrapper && f) || !specf) - return f; - else - return specf; -} - - -extern "C" JL_DLLEXPORT -void *jl_get_llvmf_decl(jl_method_instance_t *mi, size_t world, bool getwrapper, const jl_cgparams_t params) -{ - if (jl_is_method(mi->def.method) && mi->def.method->source == NULL && - mi->def.method->generator == NULL) { - // not a generic function - return NULL; + // not invalid, consider if specialized signature is worthwhile + if (prefer_specsig) + return std::make_pair(true, false); + if (!deserves_retbox(rettype) && !jl_is_datatype_singleton((jl_datatype_t*)rettype)) + return std::make_pair(true, false); + if (jl_is_uniontype(rettype)) { + bool allunbox; + size_t nbytes, align, min_align; + union_alloca_type((jl_uniontype_t*)rettype, allunbox, nbytes, align, min_align); + if (nbytes > 0) + return std::make_pair(true, false); // some elements of the union could be returned unboxed avoiding allocation } - - // compile this normally - jl_code_info_t *src = NULL; - if (jl_rettype_inferred(mi, world, world) == jl_nothing) - src = jl_type_infer(mi, world, 0); - jl_code_instance_t *codeinst = jl_compile_linfo(mi, src, world, ¶ms); - if (codeinst == NULL) - // internal error - return NULL; - - jl_llvm_functions_t decls = codeinst->functionObjectsDecls; - if (decls.functionObject == NULL && codeinst->invoke == jl_fptr_const_return && jl_is_method(mi->def.method)) { - // normally we don't generate native code for these functions, so need an exception here - // This leaks a bit of memory to cache native code that we'll never actually need - JL_LOCK(&codegen_lock); - if (decls.functionObject == NULL) { - if (src == NULL) - src = jl_type_infer(mi, world, 0); - if (src == NULL) - src = mi->def.method->generator ? jl_code_for_staged(mi) : (jl_code_info_t*)mi->def.method->source; - codeinst = jl_compile_linfo(mi, src, world, ¶ms); - if (codeinst == NULL) - // internal error - return NULL; - decls = codeinst->functionObjectsDecls; - } - JL_UNLOCK(&codegen_lock); - } - - if (!getwrapper && decls.specFunctionObject) { - if (!strcmp(decls.functionObject, "jl_fptr_args")) { - auto f = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, decls.specFunctionObject); - add_return_attr(f, Attribute::NonNull); - f->addFnAttr(Thunk); - return f; - } - else if (!strcmp(decls.functionObject, "jl_fptr_sparam")) { - auto f = Function::Create(jl_func_sig_sparams, GlobalVariable::ExternalLinkage, decls.specFunctionObject); - add_return_attr(f, Attribute::NonNull); - f->addFnAttr(Thunk); - return f; - } - else { - jl_returninfo_t returninfo = get_specsig_function(NULL, decls.specFunctionObject, mi->specTypes, codeinst->rettype); - return returninfo.decl; + bool allSingleton = true; + for (size_t i = 0; i < jl_nparams(sig); i++) { + jl_value_t *sigt = jl_tparam(sig, i); + bool issing = jl_is_datatype(sigt) && jl_is_datatype_singleton((jl_datatype_t*)sigt); + allSingleton &= issing; + if (!deserves_argbox(sigt) && !issing) { + return std::make_pair(true, false); } } - assert(decls.functionObject); - auto f = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, decls.functionObject); - add_return_attr(f, Attribute::NonNull); - f->addFnAttr(Thunk); - return f; + if (allSingleton) + return std::make_pair(true, false); + return std::make_pair(false, false); // jlcall sig won't require any box allocations } -// get a native disassembly for f (an LLVM function) -// warning: this takes ownership of, and destroys, f -extern "C" JL_DLLEXPORT -const jl_value_t *jl_dump_function_asm(void *f, int raw_mc, const char* asm_variant, const char *debuginfo) -{ - Function *llvmf = dyn_cast_or_null((Function*)f); - if (!llvmf) - jl_error("jl_dump_function_asm: Expected Function*"); - uint64_t fptr = getAddressForFunction(llvmf->getName()); - // Look in the system image as well - if (fptr == 0) - fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(llvmf); - delete llvmf; - return jl_dump_fptr_asm(fptr, raw_mc, asm_variant, debuginfo); -} // Logging for code coverage and memory allocation @@ -2357,7 +1757,7 @@ static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t arg2) { bool isboxed; - Type *at = julia_type_to_llvm(arg1.typ, &isboxed); + Type *at = julia_type_to_llvm(ctx, arg1.typ, &isboxed); assert(jl_is_datatype(arg1.typ) && arg1.typ == arg2.typ && !isboxed); if (type_is_ghost(at)) @@ -2426,7 +1826,7 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a // TODO: should we replace this with `subAns = emit_f_is(emit_getfield_knownidx(ctx, arg1, i, arg1.typ), emit_getfield_knownidx(ctx, arg2, i, arg2.typ))` // or is there any value in all this extra effort?? jl_value_t *fldty = jl_svecref(types, i); - if (type_is_ghost(julia_type_to_llvm(fldty))) + if (type_is_ghost(julia_type_to_llvm(ctx, fldty))) continue; unsigned byte_offset = jl_field_offset(sty, i); Value *subAns, *fld1, *fld2; @@ -3190,13 +2590,14 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, } -static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_code_instance_t *codeinst, StringRef specFunctionObject, - jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) +static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, + jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { // emit specialized call site - jl_value_t *jlretty = codeinst->rettype; - jl_returninfo_t returninfo = get_specsig_function(jl_Module, specFunctionObject, codeinst->def->specTypes, jlretty); + jl_returninfo_t returninfo = get_specsig_function(ctx, jl_Module, specFunctionObject, mi->specTypes, jlretty); FunctionType *cft = returninfo.decl->getFunctionType(); + *cc = returninfo.cc; + *return_roots = returninfo.return_roots; size_t nfargs = cft->getNumParams(); Value **argvals = (Value**)alloca(nfargs * sizeof(Value*)); @@ -3229,9 +2630,9 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_code_instance_t } for (size_t i = 0; i < nargs; i++) { - jl_value_t *jt = jl_nth_slot_type(codeinst->def->specTypes, i); + jl_value_t *jt = jl_nth_slot_type(mi->specTypes, i); bool isboxed = deserves_argbox(jt); - Type *et = isboxed ? T_prjlvalue : julia_type_to_llvm(jt); + Type *et = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, jt); if (type_is_ghost(et)) continue; assert(idx < nfargs); @@ -3340,25 +2741,64 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (lival.constant) { jl_method_instance_t *mi = (jl_method_instance_t*)lival.constant; assert(jl_is_method_instance(mi)); - jl_code_instance_t *codeinst = jl_compile_linfo(mi, NULL, ctx.world, ctx.params); - if (codeinst && codeinst->inferred) { - JL_GC_PUSH1(&codeinst); - const jl_llvm_functions_t &decls = codeinst->functionObjectsDecls; - if (codeinst->invoke == jl_fptr_const_return) { - assert(codeinst->rettype_const); - return mark_julia_const(codeinst->rettype_const); - } - if (decls.functionObject) { - if (!strcmp(decls.functionObject, "jl_fptr_args")) { - result = emit_call_specfun_boxed(ctx, decls.specFunctionObject, argv, nargs, rt); + if (mi == ctx.linfo) { + // handle self-recursion specially + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; + FunctionType *ft = ctx.f->getFunctionType(); + StringRef protoname = ctx.f->getName(); + if (ft == jl_func_sig) { + result = emit_call_specfun_boxed(ctx, protoname, argv, nargs, rt); + handled = true; + } + else if (ft != jl_func_sig_sparams) { + unsigned return_roots = 0; + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, argv, nargs, &cc, &return_roots, rt); + handled = true; + } + } + else { + jl_value_t *ci = jl_rettype_inferred(mi, ctx.world, ctx.world); // TODO: need to use the right pair world here + jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; + if (ci != jl_nothing && codeinst->invoke != jl_fptr_sparam) { // check if we know we definitely can't handle this specptr + if (codeinst->invoke == jl_fptr_const_return) { + result = mark_julia_const(codeinst->rettype_const); handled = true; } - else if (!!strcmp(decls.functionObject, "jl_fptr_sparam")) { - result = emit_call_specfun_other(ctx, codeinst, decls.specFunctionObject, argv, nargs, rt); + else { + bool specsig, needsparams; + std::tie(specsig, needsparams) = uses_specsig(mi, codeinst->rettype, ctx.params->prefer_specsig); + std::string name; + StringRef protoname; + bool need_to_emit = true; + if (ctx.use_cache) { + // optimization: emit the correct name immediately, if we know it + // TODO: use `emitted` map here too to try to consolidate names? + if (codeinst->specptr.fptr) { + if (specsig ? codeinst->isspecsig : codeinst->invoke == jl_fptr_args) { + protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + need_to_emit = false; + } + } + } + if (need_to_emit) { + std::stringstream namestream; + namestream << (specsig ? "j_" : "j1_") << name_from_method_instance(mi) << "_" << globalUnique++; + name = namestream.str(); + protoname = StringRef(name); + } + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; + unsigned return_roots = 0; + if (specsig) + result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, argv, nargs, &cc, &return_roots, rt); + else + result = emit_call_specfun_boxed(ctx, protoname, argv, nargs, rt); handled = true; + if (need_to_emit) { + Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); + ctx.call_targets.push_back(std::make_tuple(codeinst, cc, return_roots, trampoline_decl, specsig)); + } } } - JL_GC_POP(); } } if (!handled) { @@ -3725,7 +3165,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu Value *dest = vi.value.V; if (vi.pTIndex) ctx.builder.CreateStore(UndefValue::get(cast(vi.value.V)->getAllocatedType()), vi.value.V); - Type *store_ty = julia_type_to_llvm(rval_info.constant ? jl_typeof(rval_info.constant) : rval_info.typ); + Type *store_ty = julia_type_to_llvm(ctx, rval_info.constant ? jl_typeof(rval_info.constant) : rval_info.typ); Type *dest_ty = store_ty->getPointerTo(); if (dest_ty != dest->getType()) dest = emit_bitcast(ctx, dest, dest_ty); @@ -3813,7 +3253,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) } } bool isboxed = !deserves_stack(phiType); - Type *vtype = isboxed ? T_prjlvalue : julia_type_to_llvm(phiType); + Type *vtype = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, phiType); // The frontend should really not emit this, but we allow it // for convenience. if (type_is_ghost(vtype)) { @@ -4334,7 +3774,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) vals.push_back(ai.Vboxed); } else if (!jl_is_pointerfree(ai.typ)) { - Type *at = julia_type_to_llvm(ai.typ); + Type *at = julia_type_to_llvm(ctx, ai.typ); vals.push_back(emit_unbox(ctx, at, ai, ai.typ)); } } @@ -4403,15 +3843,51 @@ static void emit_last_age_field(jl_codectx_t &ctx) ConstantInt::get(T_size, offsetof(jl_tls_states_t, world_age) / sizeof(size_t))); } +static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_codegen_params_t ¶ms) +{ + jl_codectx_t ctx(jl_LLVMContext, params); + std::stringstream name; + name << "tojlinvoke" << globalUnique++; + Function *f = Function::Create(jl_func_sig, + GlobalVariable::PrivateLinkage, + name.str(), M); + jl_init_function(f); + //f->setAlwaysInline(); + ctx.f = f; // for jl_Module + BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", f); + ctx.builder.SetInsertPoint(b0); + Value *theFptr; + Value *theFarg; + if (codeinst->invoke != NULL) { + StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->invoke, codeinst); + theFptr = M->getOrInsertFunction(theFptrName, jlinvoke_func->getFunctionType()) +#if JL_LLVM_VERSION >= 90000 + .getCallee(); +#else + ; +#endif + theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst); + } + else { + theFptr = prepare_call(jlinvoke_func); + theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst->def); + } + theFarg = maybe_decay_untracked(theFarg); + auto args = f->arg_begin(); + Value *r = ctx.builder.CreateCall(theFptr, { &*args, &*++args, &*++args, theFarg }); + ctx.builder.CreateRet(r); + return f; +} + static void emit_cfunc_invalidate( Function *gf_thunk, jl_returninfo_t::CallingConv cc, unsigned return_roots, - jl_code_instance_t *codeinst, size_t nargs, size_t world) + jl_value_t *calltype, jl_value_t *rettype, + size_t nargs, + jl_codegen_params_t ¶ms, + Function *target=jlapplygeneric_func) { - jl_method_instance_t *lam = codeinst->def; - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = gf_thunk; - ctx.world = world; - ctx.params = &jl_default_cgparams; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", gf_thunk); ctx.builder.SetInsertPoint(b0); @@ -4426,9 +3902,9 @@ static void emit_cfunc_invalidate( if (return_roots) ++AI; for (size_t i = 0; i < nargs; i++) { - jl_value_t *jt = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *jt = jl_nth_slot_type(calltype, i); bool isboxed = deserves_argbox(jt); - Type *et = isboxed ? T_prjlvalue : julia_type_to_llvm(jt); + Type *et = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, jt); if (type_is_ghost(et)) { assert(jl_is_datatype(jt) && ((jl_datatype_t*)jt)->instance); myargs[i] = mark_julia_const(((jl_datatype_t*)jt)->instance); @@ -4448,11 +3924,10 @@ static void emit_cfunc_invalidate( } } assert(AI == gf_thunk->arg_end()); - Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, nullptr, myargs, nargs, JLCALL_F_CC); + Value *gf_ret = emit_jlcall(ctx, target, nullptr, myargs, nargs, JLCALL_F_CC); jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type); - jl_value_t *astrt = codeinst->rettype; if (cc != jl_returninfo_t::Boxed) { - emit_typecheck(ctx, gf_retbox, astrt, "cfunction"); + emit_typecheck(ctx, gf_retbox, rettype, "cfunction"); } switch (cc) { @@ -4466,21 +3941,21 @@ static void emit_cfunc_invalidate( } else { gf_ret = emit_bitcast(ctx, gf_ret, gfrt->getPointerTo()); - ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, julia_alignment(astrt))); + ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, julia_alignment(rettype))); } break; } case jl_returninfo_t::SRet: { if (return_roots) ctx.builder.CreateStore(gf_ret, gf_thunk->arg_begin() + 1); - emit_memcpy(ctx, &*gf_thunk->arg_begin(), nullptr, gf_ret, nullptr, jl_datatype_size(astrt), julia_alignment(astrt)); + emit_memcpy(ctx, &*gf_thunk->arg_begin(), nullptr, gf_ret, nullptr, jl_datatype_size(rettype), julia_alignment(rettype)); ctx.builder.CreateRetVoid(); break; } case jl_returninfo_t::Union: { Type *retty = gf_thunk->getReturnType(); Value *gf_retval = UndefValue::get(retty); - Value *tindex = compute_box_tindex(ctx, emit_typeof_boxed(ctx, gf_retbox), (jl_value_t*)jl_any_type, astrt); + Value *tindex = compute_box_tindex(ctx, emit_typeof_boxed(ctx, gf_retbox), (jl_value_t*)jl_any_type, rettype); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); gf_retval = ctx.builder.CreateInsertValue(gf_retval, gf_ret, 0); gf_retval = ctx.builder.CreateInsertValue(gf_retval, tindex, 1); @@ -4488,7 +3963,7 @@ static void emit_cfunc_invalidate( break; } case jl_returninfo_t::Ghosts: { - Value *gf_retval = compute_tindex_unboxed(ctx, gf_retbox, astrt); + Value *gf_retval = compute_tindex_unboxed(ctx, gf_retbox, rettype); ctx.builder.CreateRet(gf_retval); break; } @@ -4496,49 +3971,49 @@ static void emit_cfunc_invalidate( } static Function* gen_cfun_wrapper( - Module *into, + Module *into, jl_codegen_params_t ¶ms, const function_sig_t &sig, jl_value_t *ff, - jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt, + jl_typemap_entry_t *sf, jl_value_t *declrt, jl_method_instance_t *lam, jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types) { // Generate a c-callable wrapper size_t nargs = sig.nccallargs; const char *name = "cfunction"; size_t world = jl_world_counter; - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; - bool nest = (!ff || unionall_env); - // try to look up this function for direct invoking - jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 1) : NULL; jl_code_instance_t *codeinst = NULL; + bool nest = (!ff || unionall_env); jl_value_t *astrt = (jl_value_t*)jl_any_type; + void *callptr = NULL; + int calltype = 0; // infer it first, if necessary - if (lam) { + // FIXME! pretend this is OK + if (lam) name = jl_symbol_name(lam->def.method->name); - jl_code_info_t *src = NULL; - if (!into && jl_rettype_inferred(lam, world, world) == jl_nothing) // TODO: this isn't ideal to be unconditionally calling type inference from here - src = jl_type_infer(lam, world, 0); - codeinst = jl_compile_linfo(lam, src, world, &jl_default_cgparams); - if (codeinst) { - if (!codeinst->inferred || - codeinst->functionObjectsDecls.specFunctionObject == NULL || - !strcmp(codeinst->functionObjectsDecls.specFunctionObject, "jl_fptr_sparam")) { - codeinst = NULL; // TODO: use emit_invoke framework to dispatch these - } - } - if (codeinst) { - astrt = codeinst->rettype; - if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function does not return since it is - // occasionally required by the C API (typically error callbacks) - // and doesn't capture the majority of the case when a function - // may throw. - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name); - } - } - else { - lam = NULL; + if (lam && params.cache) { + // if (!into) + // TODO: this isn't ideal to be unconditionally calling type inference (and compile) from here + codeinst = jl_compile_method_internal(lam, world); + assert(codeinst->invoke); + if (codeinst->invoke == jl_fptr_args) { + callptr = codeinst->specptr.fptr; + calltype = 1; + } + else if (codeinst->invoke == jl_fptr_const_return) { + // don't need the fptr + callptr = (void*)codeinst->rettype_const; + calltype = 2; + } + else if (codeinst->isspecsig) { + callptr = codeinst->specptr.fptr; + calltype = 3; + } + astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is + // occasionally required by the C API (typically error callbacks) + // even though we're likely to encounter memory errors in that case + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name); } } @@ -4584,11 +4059,9 @@ static Function* gen_cfun_wrapper( JL_GC_POP(); } - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = cw; - ctx.linfo = lam; ctx.world = world; - ctx.params = &jl_default_cgparams; ctx.name = name; ctx.funcName = name; @@ -4611,7 +4084,7 @@ static Function* gen_cfun_wrapper( Value *world_v = ctx.builder.CreateLoad(prepare_global(jlgetworld_global)); Value *age_ok = NULL; - if (lam) { + if (calltype) { Value *lam_max = ctx.builder.CreateLoad( T_size, ctx.builder.CreateConstInBoundsGEP1_32( @@ -4679,7 +4152,7 @@ static Function* gen_cfun_wrapper( } else if (static_at && jl_is_concrete_immutable(jargty)) { // anything that could be stored unboxed bool isboxed; - Type *T = julia_type_to_llvm(jargty, &isboxed); + Type *T = julia_type_to_llvm(ctx, jargty, &isboxed); assert(!isboxed); // a T* (of unknown origin) if (type_is_ghost(T)) { @@ -4756,7 +4229,7 @@ static Function* gen_cfun_wrapper( if (static_at) { bool isboxed; assert(jargty == jargty_proper); - (void)julia_type_to_llvm(jargty, &isboxed); + (void)julia_type_to_llvm(ctx, jargty, &isboxed); if (isboxed) inputarg = mark_julia_type(ctx, box_ccall_result(ctx, val, literal_pointer_val(ctx, jargty), jargty), @@ -4782,31 +4255,28 @@ static Function* gen_cfun_wrapper( // Create the call bool jlfunc_sret; jl_cgval_t retval; - if (codeinst && codeinst->invoke == jl_fptr_const_return) { + if (calltype == 2) { nargs = 0; // arguments not needed -- TODO: not really true, should emit an age_ok test and jlcall jlfunc_sret = false; - retval = mark_julia_const(codeinst->rettype_const); + retval = mark_julia_const((jl_value_t*)callptr); } - else if (!codeinst || !codeinst->functionObjectsDecls.functionObject || - !strcmp(codeinst->functionObjectsDecls.functionObject, "jl_fptr_args") || - !strcmp(codeinst->functionObjectsDecls.functionObject, "jl_fptr_sparam")) { + else if (calltype == 0 || calltype == 1) { // emit a jlcall jlfunc_sret = false; Function *theFptr = NULL; - if (codeinst && codeinst->functionObjectsDecls.functionObject) { - if (!strcmp(codeinst->functionObjectsDecls.functionObject, "jl_fptr_args")) { - const char *fname = codeinst->functionObjectsDecls.specFunctionObject; - theFptr = cast_or_null(jl_Module->getNamedValue(fname)); - if (!theFptr) { - theFptr = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, - fname, jl_Module); - } - else { - assert(theFptr->getFunctionType() == jl_func_sig); - } - add_return_attr(theFptr, Attribute::NonNull); - theFptr->addFnAttr(Thunk); + if (calltype == 1) { + StringRef fname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, codeinst); + theFptr = cast_or_null(jl_Module->getNamedValue(fname)); + if (!theFptr) { + theFptr = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, + fname, jl_Module); + jl_init_function(theFptr); + } + else { + assert(theFptr->getFunctionType() == jl_func_sig); } + add_return_attr(theFptr, Attribute::NonNull); + theFptr->addFnAttr(Thunk); } BasicBlock *b_generic, *b_jlcall, *b_after; Value *ret_jlcall; @@ -4836,9 +4306,10 @@ static Function* gen_cfun_wrapper( retval = mark_julia_type(ctx, ret, true, astrt); } else { + assert(calltype == 3); // emit a specsig call - const char *protoname = codeinst->functionObjectsDecls.specFunctionObject; - jl_returninfo_t returninfo = get_specsig_function(M, protoname, lam->specTypes, codeinst->rettype); + StringRef protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, codeinst); + jl_returninfo_t returninfo = get_specsig_function(ctx, M, protoname, lam->specTypes, astrt); FunctionType *cft = returninfo.decl->getFunctionType(); jlfunc_sret = (returninfo.cc == jl_returninfo_t::SRet); @@ -4865,7 +4336,7 @@ static Function* gen_cfun_wrapper( Value *arg; jl_value_t *spect = jl_nth_slot_type(lam->specTypes, i); bool isboxed = deserves_argbox(spect); - Type *T = isboxed ? T_prjlvalue : julia_type_to_llvm(spect); + Type *T = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, spect); if (isboxed) { arg = boxed(ctx, inputarg); } @@ -4898,7 +4369,7 @@ static Function* gen_cfun_wrapper( // build a specsig -> jl_apply_generic converter thunk // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), // but which has the signature of a specsig - emit_cfunc_invalidate(gf_thunk, returninfo.cc, returninfo.return_roots, codeinst, nargs + 1, world); + emit_cfunc_invalidate(gf_thunk, returninfo.cc, returninfo.return_roots, lam->specTypes, codeinst->rettype, nargs + 1, ctx.emission_context); theFptr = ctx.builder.CreateSelect(age_ok, theFptr, gf_thunk); } CallInst *call = ctx.builder.CreateCall(theFptr, ArrayRef(args)); @@ -4988,7 +4459,7 @@ static Function* gen_cfun_wrapper( } if (!into) - jl_finalize_module(M, true); + jl_finalize_module(std::unique_ptr(M)); return cw_proto; } @@ -5034,6 +4505,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con /* inputs: */ rt, (jl_value_t*)argt, unionall_env, sparam_vals, + &ctx.emission_context, /* outputs: */ lrt, retboxed, static_rt); if (!err.empty()) { @@ -5044,7 +4516,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con if (rt != declrt && rt != (jl_value_t*)jl_any_type) jl_add_method_root(ctx, rt); - function_sig_t sig("cfunction", lrt, rt, retboxed, argt, unionall_env, false, CallingConv::C, false); + function_sig_t sig("cfunction", lrt, rt, retboxed, argt, unionall_env, false, CallingConv::C, false, &ctx.emission_context); assert(sig.fargt.size() + sig.sret == sig.fargt_sig.size()); if (!sig.err_msg.empty()) { emit_error(ctx, sig.err_msg); @@ -5092,10 +4564,15 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } #endif + size_t world = jl_world_counter; + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + // try to look up this function for direct invoking + jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0) : NULL; Value *F = gen_cfun_wrapper( - jl_Module, + jl_Module, ctx.emission_context, sig, fexpr_rt.constant, - NULL, declrt, (jl_tupletype_t*)sigt, + NULL, declrt, lam, unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { @@ -5161,7 +4638,8 @@ const struct jl_typemap_info cfunction_cache = { jl_array_t *jl_cfunction_list; -static Function *jl_cfunction_object(jl_value_t *ff, jl_value_t *declrt, jl_tupletype_t *argt) +Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_tupletype_t *argt, + jl_codegen_params_t ¶ms) { // Assumes the codegen lock is acquired. The caller is responsible for that. jl_ptls_t ptls = jl_get_ptls_states(); @@ -5227,7 +4705,7 @@ static Function *jl_cfunction_object(jl_value_t *ff, jl_value_t *declrt, jl_tupl crt = (jl_value_t*)jl_any_type; } bool toboxed; - Type *lcrt = julia_struct_to_llvm(crt, NULL, &toboxed); + Type *lcrt = _julia_struct_to_llvm(¶ms, crt, NULL, &toboxed); if (lcrt == NULL) jl_error("cfunction: return type doesn't correspond to a C type"); else if (toboxed) @@ -5256,9 +4734,14 @@ static Function *jl_cfunction_object(jl_value_t *ff, jl_value_t *declrt, jl_tupl jl_value_t *err; { // scope block for sig function_sig_t sig("cfunction", lcrt, crt, toboxed, - argt->parameters, NULL, false, CallingConv::C, false); + argt->parameters, NULL, false, CallingConv::C, false, ¶ms); if (sig.err_msg.empty()) { - Function *F = gen_cfun_wrapper(NULL, sig, ff, cache_l3, declrt, (jl_tupletype_t*)sigt, NULL, NULL, NULL); + size_t world = jl_world_counter; + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + // try to look up this function for direct invoking + jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0); + Function *F = gen_cfun_wrapper(NULL, params, sig, ff, cache_l3, declrt, lam, NULL, NULL, NULL); JL_GC_POP(); return F; } @@ -5268,7 +4751,8 @@ static Function *jl_cfunction_object(jl_value_t *ff, jl_value_t *declrt, jl_tupl } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, const jl_returninfo_t &f, int retarg, StringRef funcName, Module *M) +static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, const jl_returninfo_t &f, int retarg, StringRef funcName, + Module *M, jl_codegen_params_t ¶ms) { Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, funcName, M); add_return_attr(w, Attribute::NonNull); @@ -5281,11 +4765,11 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret //Value *mfunc = &*AI++; (void)mfunc; // unused assert(AI == w->arg_end()); - jl_codectx_t ctx(jl_LLVMContext); + jl_codectx_t ctx(jl_LLVMContext, params); ctx.f = w; ctx.linfo = lam; + ctx.rettype = jlretty; ctx.world = 0; - ctx.params = &jl_default_cgparams; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", w); ctx.builder.SetInsertPoint(b0); @@ -5327,7 +4811,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret for (size_t i = 0; i < jl_nparams(lam->specTypes) && idx < nfargs; ++i) { jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); bool isboxed = deserves_argbox(ty); - Type *lty = isboxed ? T_prjlvalue : julia_type_to_llvm(ty); + Type *lty = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, ty); if (type_is_ghost(lty)) continue; Value *theArg; @@ -5389,49 +4873,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret return w; } -static bool uses_specsig(jl_value_t *sig, size_t nreq, jl_value_t *rettype, bool needsparam, bool va, jl_code_info_t *src, bool prefer_specsig) -{ - if (needsparam) - return false; - if (!src || !jl_ast_flag_inferred((jl_array_t*)src)) - return false; - if (sig == (jl_value_t*)jl_anytuple_type) - return false; - if (!jl_is_datatype(sig)) - return false; - if (jl_nparams(sig) == 0) - return false; - if (va) { - if (jl_is_vararg_type(jl_tparam(sig, jl_nparams(sig)-1))) - return false; - } - // not invalid, consider if specialized signature is worthwhile - if (prefer_specsig) - return true; - if (!deserves_retbox(rettype) && !jl_is_datatype_singleton((jl_datatype_t*)rettype)) - return true; - if (jl_is_uniontype(rettype)) { - bool allunbox; - size_t nbytes, align, min_align; - union_alloca_type((jl_uniontype_t*)rettype, allunbox, nbytes, align, min_align); - if (nbytes > 0) - return true; // some elements of the union could be returned unboxed avoiding allocation - } - bool allSingleton = true; - for (size_t i = 0; i < jl_nparams(sig); i++) { - jl_value_t *sigt = jl_tparam(sig, i); - bool issing = jl_is_datatype(sigt) && jl_is_datatype_singleton((jl_datatype_t*)sigt); - allSingleton &= issing; - if (!deserves_argbox(sigt) && !issing) { - return true; - } - } - if (allSingleton) - return true; - return false; // jlcall sig won't require any box allocations -} - -static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype) +static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype) { jl_returninfo_t props = {}; SmallVector fsig; @@ -5460,7 +4902,7 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, } else if (!deserves_retbox(jlrettype)) { bool retboxed; - rt = julia_type_to_llvm(jlrettype, &retboxed); + rt = julia_type_to_llvm(ctx, jlrettype, &retboxed); assert(!retboxed); if (rt != T_void && deserves_sret(jlrettype, rt)) { auto tracked = CountTrackedPointers(rt); @@ -5501,7 +4943,7 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, for (size_t i = 0; i < jl_nparams(sig); i++) { jl_value_t *jt = jl_tparam(sig, i); - Type *ty = deserves_argbox(jt) ? T_prjlvalue : julia_type_to_llvm(jt); + Type *ty = deserves_argbox(jt) ? T_prjlvalue : julia_type_to_llvm(ctx, jt); if (type_is_ghost(ty)) continue; unsigned argno = fsig.size(); @@ -5518,6 +4960,7 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, if (f == NULL) { f = Function::Create(ftype, GlobalVariable::ExternalLinkage, name, M); f->setAttributes(attributes); + jl_init_function(f); } else { assert(f->getFunctionType() == ftype); @@ -5539,13 +4982,14 @@ static void emit_sret_roots(jl_codectx_t &ctx, bool isptr, Value *Src, Type *T, } static DISubroutineType * -get_specsig_di(jl_value_t *rt, jl_value_t *sig, DIFile *topfile, DIBuilder &dbuilder) +get_specsig_di(jl_codectx_t &ctx, jl_value_t *rt, jl_value_t *sig, DIBuilder &dbuilder) { - std::vector ditypes(0); - ditypes.push_back(julia_type_to_di(rt, &dbuilder, false)); - for (size_t i = 0; i < jl_nparams(sig); i++) { + size_t nargs = jl_nparams(sig); // TODO: if this is a Varargs function, our debug info for the `...` var may be misleading + std::vector ditypes(nargs + 1); + ditypes[0] = julia_type_to_di(ctx, rt, &dbuilder, false); + for (size_t i = 0; i < nargs; i++) { jl_value_t *jt = jl_tparam(sig, i); - ditypes.push_back(julia_type_to_di(jt, &dbuilder, false)); + ditypes[i + 1] = julia_type_to_di(ctx, jt, &dbuilder, false); } return dbuilder.createSubroutineType(dbuilder.getOrCreateTypeArray(ditypes)); } @@ -5565,31 +5009,28 @@ static jl_datatype_t *compute_va_type(jl_method_instance_t *lam, size_t nreq) } // Compile to LLVM IR, using a specialized signature if applicable. -static std::unique_ptr emit_function( +static std::pair, jl_llvm_functions_t> + emit_function( jl_method_instance_t *lam, jl_code_info_t *src, jl_value_t *jlrettype, - size_t world, - jl_llvm_functions_t *declarations, - const jl_cgparams_t *params) + jl_codegen_params_t ¶ms) { - assert(declarations && "Capturing declarations is always required"); - // step 1. unpack AST and allocate codegen context for this function - jl_codectx_t ctx(jl_LLVMContext); + jl_llvm_functions_t declarations; + jl_codectx_t ctx(jl_LLVMContext, params); JL_GC_PUSH2(&ctx.code, &ctx.roots); - ctx.code = (jl_array_t*)src->code; + ctx.code = src->code; //jl_static_show(JL_STDOUT, (jl_value_t*)ast); //jl_printf(JL_STDOUT, "\n"); std::map labels; ctx.module = jl_is_method(lam->def.method) ? lam->def.method->module : lam->def.module; ctx.linfo = lam; + ctx.rettype = jlrettype; ctx.source = src; - ctx.world = world; ctx.name = name_from_method_instance(lam); ctx.funcName = ctx.name; - ctx.params = params; ctx.spvals_ptr = NULL; ctx.nargs = jl_is_method(lam->def.method) ? lam->def.method->nargs : 0; bool toplevel = !jl_is_method(lam->def.method); @@ -5597,8 +5038,8 @@ static std::unique_ptr emit_function( size_t stmtslen = jl_array_dim0(stmts); if (JL_HOOK_TEST(ctx.params, emit_function)) { - JL_HOOK_CALL(ctx.params, emit_function, 3, (jl_value_t*)ctx.linfo, - (jl_value_t*)ctx.source, jl_box_ulong(world)); + JL_HOOK_CALL(ctx.params, emit_function, 2, (jl_value_t*)ctx.linfo, + (jl_value_t*)ctx.source); } // step 1b. unpack debug information @@ -5653,17 +5094,10 @@ static std::unique_ptr emit_function( ctx.ssavalue_assigned.assign(n_ssavalues, false); ctx.SAvalues.assign(n_ssavalues, jl_cgval_t()); - bool needsparams = false; - if (jl_is_method(lam->def.method)) { - if ((size_t)jl_subtype_env_size(lam->def.method->sig) != jl_svec_len(lam->sparam_vals)) - needsparams = true; - for (size_t i = 0; i < jl_svec_len(lam->sparam_vals); ++i) { - if (jl_is_typevar(jl_svecref(lam->sparam_vals, i))) - needsparams = true; - } - } - - bool specsig = uses_specsig(lam->specTypes, nreq, jlrettype, needsparams, va, src, params->prefer_specsig); + bool specsig, needsparams; + std::tie(specsig, needsparams) = uses_specsig(lam, jlrettype, params.params->prefer_specsig); + if (!src->inferred) + specsig = false; // step 3. some variable analysis size_t i; @@ -5718,15 +5152,16 @@ static std::unique_ptr emit_function( unadorned_name++; #endif funcName << unadorned_name << "_" << globalUnique++; + declarations.specFunctionObject = funcName.str(); // allocate Function declarations and wrapper objects Module *M = new Module(ctx.name, jl_LLVMContext); - jl_setup_module(M, params); + jl_setup_module(M, ctx.params); jl_returninfo_t returninfo = {}; Function *f = NULL; bool has_sret = false; if (specsig) { // assumes !va and !needsparams - returninfo = get_specsig_function(M, funcName.str(), lam->specTypes, jlrettype); + returninfo = get_specsig_function(ctx, M, declarations.specFunctionObject, lam->specTypes, jlrettype); f = returninfo.decl; has_sret = (returninfo.cc == jl_returninfo_t::SRet || returninfo.cc == jl_returninfo_t::Union); jl_init_function(f); @@ -5755,23 +5190,22 @@ static std::unique_ptr emit_function( std::stringstream wrapName; wrapName << "jfptr_" << unadorned_name << "_" << globalUnique; - Function *fwrap = gen_invoke_wrapper(lam, jlrettype, returninfo, retarg, wrapName.str(), M); - declarations->functionObject = strdup(fwrap->getName().str().c_str()); + declarations.functionObject = wrapName.str(); + (void)gen_invoke_wrapper(lam, jlrettype, returninfo, retarg, declarations.functionObject, M, ctx.emission_context); } else { f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, GlobalVariable::ExternalLinkage, - funcName.str(), M); + declarations.specFunctionObject, M); + jl_init_function(f); add_return_attr(f, Attribute::NonNull); f->addFnAttr(Thunk); // TODO: (if needsparams) add attributes: dereferenceable, readonly, nocapture // TODO: add attributes: dereferenceable, readonly, nocapture - e.g. maybe_mark_argument_dereferenceable(Arg, argType); // TODO: add attributes: dereferenceable, readonly, nocapture returninfo.decl = f; - jl_init_function(f); - declarations->functionObject = needsparams ? "jl_fptr_sparam" : "jl_fptr_args"; + declarations.functionObject = needsparams ? "jl_fptr_sparam" : "jl_fptr_args"; } - declarations->specFunctionObject = strdup(f->getName().str().c_str()); if (jlrettype == (jl_value_t*)jl_bottom_type) f->setDoesNotReturn(); @@ -5835,7 +5269,7 @@ static std::unique_ptr emit_function( subrty = jl_di_func_sig; } else { - subrty = get_specsig_di(jlrettype, lam->specTypes, topfile, dbuilder); + subrty = get_specsig_di(ctx, jlrettype, lam->specTypes, dbuilder); } SP = dbuilder.createFunction(CU ,dbgFuncName // Name @@ -5844,7 +5278,7 @@ static std::unique_ptr emit_function( ,toplineno // LineNo ,subrty // Ty ,toplineno // ScopeLine - ,DIFlagZero // Flags + ,DINode::FlagZero // Flags ,DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized // SPFlags ,nullptr // Template Parameters ,nullptr // Template Declaration @@ -5867,9 +5301,9 @@ static std::unique_ptr emit_function( topfile, // File toplineno == -1 ? 0 : toplineno, // Line // Variable type - julia_type_to_di(varinfo.value.typ, &dbuilder, false), + julia_type_to_di(ctx, varinfo.value.typ, &dbuilder, false), AlwaysPreserve, // May be deleted if optimized out - DIFlagZero); // Flags (TODO: Do we need any) + DINode::FlagZero); // Flags (TODO: Do we need any) } if (va && ctx.vaSlot != -1) { ctx.slots[ctx.vaSlot].dinfo = dbuilder.createParameterVariable( @@ -5878,9 +5312,9 @@ static std::unique_ptr emit_function( has_sret + nreq + 1, // Argument number (1-based) topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), + julia_type_to_di(ctx, ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), AlwaysPreserve, // May be deleted if optimized out - DIFlagZero); // Flags (TODO: Do we need any) + DINode::FlagZero); // Flags (TODO: Do we need any) } for (i = 0; i < vinfoslen; i++) { jl_sym_t *s = (jl_sym_t*)jl_array_ptr_ref(src->slotnames, i); @@ -5893,9 +5327,9 @@ static std::unique_ptr emit_function( jl_symbol_name(s), // Variable name topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(varinfo.value.typ, &dbuilder, false), // Variable type + julia_type_to_di(ctx, varinfo.value.typ, &dbuilder, false), // Variable type AlwaysPreserve, // May be deleted if optimized out - DIFlagZero // Flags (TODO: Do we need any) + DINode::FlagZero // Flags (TODO: Do we need any) ); } } @@ -5983,7 +5417,7 @@ static std::unique_ptr emit_function( } else if (deserves_stack(jt, true)) { bool isboxed; - Type *vtype = julia_type_to_llvm(jt, &isboxed); + Type *vtype = julia_type_to_llvm(ctx, jt, &isboxed); assert(!isboxed); assert(!type_is_ghost(vtype) && "constants should already be handled"); // CreateAlloca is OK during prologue setup @@ -6094,7 +5528,7 @@ static std::unique_ptr emit_function( jl_sym_t *s = (jl_sym_t*)jl_array_ptr_ref(src->slotnames, i); jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); bool isboxed = deserves_argbox(argType); - Type *llvmArgType = isboxed ? T_prjlvalue : julia_type_to_llvm(argType); + Type *llvmArgType = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, argType); if (s == unused_sym) { if (specsig && !type_is_ghost(llvmArgType)) ++AI; @@ -6176,17 +5610,17 @@ static std::unique_ptr emit_function( for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); bool isboxed = deserves_argbox(argType); - Type *llvmArgType = isboxed ? T_prjlvalue : julia_type_to_llvm(argType); + Type *llvmArgType = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, argType); vargs[i - nreq] = get_specsig_arg(argType, llvmArgType, isboxed); } if (jl_is_concrete_type(vi.value.typ)) { jl_cgval_t tuple = emit_new_struct(ctx, vi.value.typ, ctx.nvargs, vargs); - // FIXME: this may assert since the type of vi might not be isbits here emit_varinfo_assign(ctx, vi, tuple); - } else { - Value *vtpl = emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null), + } + else { + restTuple = emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null), vargs, ctx.nvargs, JLCALL_F_CC); - jl_cgval_t tuple = mark_julia_type(ctx, vtpl, true, vi.value.typ); + jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, vi.value.typ); emit_varinfo_assign(ctx, vi, tuple); } } @@ -6271,7 +5705,7 @@ static std::unique_ptr emit_function( ,0 // LineNo ,jl_di_func_null_sig // Ty ,0 // ScopeLine - ,DIFlagZero // Flags + ,DINode::FlagZero // Flags ,DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized // SPFlags ,nullptr // Template Parameters ,nullptr // Template Declaration @@ -6556,7 +5990,7 @@ static std::unique_ptr emit_function( if (sret) { if (retvalinfo.ispointer()) { if (returninfo.return_roots) { - Type *store_ty = julia_type_to_llvm(retvalinfo.typ); + Type *store_ty = julia_type_to_llvm(ctx, retvalinfo.typ); emit_sret_roots(ctx, true, data_pointer(ctx, retvalinfo), store_ty, f->arg_begin() + 1, returninfo.return_roots); } if (returninfo.cc == jl_returninfo_t::SRet) { @@ -6573,7 +6007,7 @@ static std::unique_ptr emit_function( Type *dest_ty = store_ty->getPointerTo(); Value *Val = retvalinfo.V; if (returninfo.return_roots) { - assert(julia_type_to_llvm(retvalinfo.typ) == store_ty); + assert(julia_type_to_llvm(ctx, retvalinfo.typ) == store_ty); emit_sret_roots(ctx, false, Val, store_ty, f->arg_begin() + 1, returninfo.return_roots); } if (dest_ty != sret->getType()) @@ -6763,7 +6197,7 @@ static std::unique_ptr emit_function( if (val.constant) val = mark_julia_const(val.constant); // be over-conservative at making sure `.typ` is set concretely, not tindex if (!jl_is_uniontype(phiType) || !TindexN) { - Type *lty = julia_type_to_llvm(phiType); + Type *lty = julia_type_to_llvm(ctx, phiType); if (VN) { Value *V; if (val.typ == (jl_value_t*)jl_bottom_type) { @@ -6799,7 +6233,7 @@ static std::unique_ptr emit_function( } else { V = ConstantPointerNull::get(cast(T_prjlvalue)); - Type *lty = julia_type_to_llvm(val.typ); + Type *lty = julia_type_to_llvm(ctx, val.typ); if (dest && !type_is_ghost(lty)) // basically, if !ghost union emit_unbox(ctx, lty, val, val.typ, dest); RTindex = ConstantInt::get(T_int8, tindex); @@ -6982,14 +6416,215 @@ static std::unique_ptr emit_function( } if (JL_HOOK_TEST(ctx.params, emitted_function)) { - JL_HOOK_CALL(ctx.params, emitted_function, 3, (jl_value_t*)ctx.linfo, - (jl_value_t*)ctx.source, jl_box_ulong(world)); + JL_HOOK_CALL(ctx.params, emitted_function, 2, (jl_value_t*)ctx.linfo, + (jl_value_t*)ctx.source); + } + + JL_GC_POP(); + return std::make_pair(std::unique_ptr(M), declarations); +} + +// --- entry point --- + +void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL); + +JL_GCC_IGNORE_START("-Wclobbered") +jl_compile_result_t jl_emit_code( + jl_method_instance_t *li, + jl_code_info_t *src, + jl_value_t *jlrettype, + jl_codegen_params_t ¶ms) +{ + // caller must hold codegen_lock + jl_llvm_functions_t decls = {}; + std::unique_ptr m; + assert((params.params == &jl_default_cgparams /* fast path */ || !params.cache || + compare_cgparams(params.params, &jl_default_cgparams)) && + "functions compiled with custom codegen params must not be cached"); + JL_TRY { + std::tie(m, decls) = emit_function(li, src, jlrettype, params); } + JL_CATCH { + // Something failed! This is very, very bad. + // Try to pretend that it isn't and attempt to recover. + m.reset(); + decls.functionObject = ""; + decls.specFunctionObject = ""; + const char *mname = name_from_method_instance(li); + jl_printf((JL_STREAM*)STDERR_FILENO, "Internal error: encountered unexpected error during compilation of %s:\n", mname); + jl_static_show((JL_STREAM*)STDERR_FILENO, jl_current_exception()); + jl_printf((JL_STREAM*)STDERR_FILENO, "\n"); + jlbacktrace(); // written to STDERR_FILENO + } + + return std::make_tuple(std::move(m), decls); +} +jl_compile_result_t jl_emit_codeinst( + jl_code_instance_t *codeinst, + jl_code_info_t *src, + jl_codegen_params_t ¶ms) +{ + JL_GC_PUSH1(&src); + if (!src) { + src = (jl_code_info_t*)codeinst->inferred; + jl_method_t *def = codeinst->def->def.method; + if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def)) + src = jl_uncompress_ast(def, codeinst, (jl_array_t*)src); + if (!src || !jl_is_code_info(src)) { + JL_GC_POP(); + return jl_compile_result_t(); // failed + } + } + jl_compile_result_t result = jl_emit_code(codeinst->def, src, codeinst->rettype, params); + + const jl_llvm_functions_t &decls = std::get<1>(result); + const std::string &specf = decls.specFunctionObject; + const std::string &f = decls.functionObject; + if (params.cache && !f.empty()) { + const Module *m = std::get<0>(result).get(); + // Prepare debug info to receive this function + // record that this function name came from this linfo, + // so we can build a reverse mapping for debug-info. + if (!JL_HOOK_TEST(params.params, module_activation)) { + bool toplevel = !jl_is_method(codeinst->def->def.method); + if (!toplevel) { + const DataLayout &DL = m->getDataLayout(); + // but don't remember toplevel thunks because + // they may not be rooted in the gc for the life of the program, + // and the runtime doesn't notify us when the code becomes unreachable :( + if (!specf.empty()) + jl_add_code_in_flight(specf, codeinst, DL); + if (!f.empty() && f != "jl_fptr_args" && f != "jl_fptr_sparam") + jl_add_code_in_flight(f, codeinst, DL); + } + } + + if (// don't alter `inferred` when the code is not directly being used + params.world && + // don't change inferred state + codeinst->inferred) { + jl_method_t *def = codeinst->def->def.method; + if (// keep code when keeping everything + !(JL_DELETE_NON_INLINEABLE) || + // aggressively keep code when debugging level >= 2 + jl_options.debug_level > 1) { + // update the stored code + if (codeinst->inferred != (jl_value_t*)src) { + if (jl_is_method(def)) + src = (jl_code_info_t*)jl_compress_ast(def, src); + codeinst->inferred = (jl_value_t*)src; + jl_gc_wb(codeinst, src); + } + } + else if (// don't delete toplevel code + jl_is_method(def) && + // and there is something to delete (test this before calling jl_ast_flag_inlineable) + codeinst->inferred != jl_nothing && + // don't delete inlineable code, unless it is constant + (codeinst->invoke == jl_fptr_const_return || !jl_ast_flag_inlineable((jl_array_t*)codeinst->inferred)) && + // don't delete code when generating a precompile file + !imaging_mode) { + // if not inlineable, code won't be needed again + codeinst->inferred = jl_nothing; + } + } + } JL_GC_POP(); - return std::unique_ptr(M); + return result; } + +void jl_compile_workqueue( + std::map &emitted, + jl_codegen_params_t ¶ms) +{ + while (!params.workqueue.empty()) { + jl_code_instance_t *codeinst; + Function *protodecl; + jl_returninfo_t::CallingConv proto_cc; + bool proto_specsig; + unsigned proto_return_roots; + std::tie(codeinst, proto_cc, proto_return_roots, protodecl, proto_specsig) = params.workqueue.back(); + params.workqueue.pop_back(); + // try to emit code for this item from the workqueue + assert(codeinst->min_world <= params.world && codeinst->max_world >= params.world && + "invalid world for code-instance"); + StringRef preal_decl = ""; + bool preal_specsig = false; + if (params.cache && codeinst->invoke != NULL) { + if (codeinst->invoke == jl_fptr_args) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + } + else if (codeinst->isspecsig) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)codeinst->specptr.fptr, codeinst); + preal_specsig = true; + } + } + else { + jl_compile_result_t &result = emitted[codeinst]; + jl_llvm_functions_t *decls = NULL; + if (std::get<0>(result)) { + decls = &std::get<1>(result); + } + else { + result = jl_emit_codeinst(codeinst, NULL, params); + if (std::get<0>(result)) + decls = &std::get<1>(result); + else + emitted.erase(codeinst); // undo the insert above + } + if (decls) { + if (decls->functionObject == "jl_fptr_args") { + preal_decl = decls->specFunctionObject; + } + else if (decls->functionObject != "jl_fptr_sparam") { + preal_decl = decls->specFunctionObject; + preal_specsig = true; + } + } + } + // patch up the prototype we emitted earlier + Module *mod = protodecl->getParent(); + assert(protodecl->isDeclaration()); + if (proto_specsig) { + // expected specsig + if (!preal_specsig) { + // emit specsig-to-(jl)invoke conversion + Function *preal = emit_tojlinvoke(codeinst, mod, params); + protodecl->setLinkage(GlobalVariable::PrivateLinkage); + //protodecl->setAlwaysInline(); + protodecl->addFnAttr("no-frame-pointer-elim", "true"); + size_t nrealargs = jl_nparams(codeinst->def->specTypes); // number of actual arguments being passed + // TODO: maybe this can be cached in codeinst->specfptr? + emit_cfunc_invalidate(protodecl, proto_cc, proto_return_roots, codeinst->def->specTypes, codeinst->rettype, nrealargs, params, preal); + preal_decl = ""; // no need to fixup the name + } + else { + assert(!preal_decl.empty()); + } + } + else { + // expected non-specsig + if (preal_decl.empty() || preal_specsig) { + // emit jlcall1-to-(jl)invoke conversion + preal_decl = emit_tojlinvoke(codeinst, mod, params)->getName(); + } + } + if (!preal_decl.empty()) { + // merge and/or rename this prototype to the real function + if (Value *specfun = mod->getNamedValue(preal_decl)) { + if (protodecl != specfun) + protodecl->replaceAllUsesWith(specfun); + } + else { + protodecl->setName(preal_decl); + } + } + } +} + + // --- initialization --- std::pair tbaa_make_child(const char *name, MDNode *parent=nullptr, bool isConstant=false) @@ -7028,40 +6663,8 @@ static GlobalVariable *julia_const_gv(jl_value_t *val) return nullptr; } -// TODO: do this lazily -extern "C" void jl_fptr_to_llvm(void *fptr, jl_code_instance_t *lam, int specsig) -{ - if (!imaging_mode) { // in imaging mode, it's fine to use the fptr, but we don't want it in the shadow_module - // this assigns a function pointer (from loading the system image), to the function object - std::stringstream funcName; - if (!specsig) - funcName << "jsys_"; // the invoke implementation wrapper - else if (lam->invoke == jl_fptr_args) - funcName << "jsys1_"; - else if (lam->invoke == jl_fptr_sparam) - funcName << "jsys3_"; - else - funcName << "julia_"; // it's a specsig call - const char* unadorned_name = jl_symbol_name(lam->def->def.method->name); - funcName << unadorned_name << "_" << globalUnique++; - Function *f = Function::Create(jl_func_sig, Function::ExternalLinkage, funcName.str()); - add_named_global(f, fptr); - const char **fdecl; - if (specsig) { - fdecl = &lam->functionObjectsDecls.specFunctionObject; - if (lam->invoke == jl_fptr_args) - lam->functionObjectsDecls.functionObject = "jl_fptr_args"; - else if (lam->invoke == jl_fptr_sparam) - lam->functionObjectsDecls.functionObject = "jl_fptr_sparam"; - } - else { - fdecl = &lam->functionObjectsDecls.functionObject; - } - assert(!*fdecl); - *fdecl = strdup(f->getName().str().c_str()); - delete f; - } -} +// TODO: delete this +extern "C" void jl_fptr_to_llvm(void *fptr, jl_code_instance_t *lam, int specsig) { } static void init_julia_llvm_meta(void) { @@ -7138,14 +6741,14 @@ static void init_julia_llvm_env(Module *m) // add needed base debugging definitions to our LLVM environment DIBuilder dbuilder(*m); - DIFile *julia_h = dbuilder.createFile("julia.h",""); + DIFile *julia_h = dbuilder.createFile("julia.h", ""); jl_value_dillvmt = dbuilder.createStructType(nullptr, "jl_value_t", julia_h, 71, // At the time of this writing. Not sure if it's worth it to keep this in sync 0 * 8, // sizeof(jl_value_t) * 8, __alignof__(void*) * 8, // __alignof__(jl_value_t) * 8, - DIFlagZero, // Flags + DINode::FlagZero, // Flags nullptr, // Derived from nullptr); // Elements - will be corrected later @@ -7158,14 +6761,14 @@ static void init_julia_llvm_env(Module *m) dbuilder.replaceArrays(jl_value_dillvmt, dbuilder.getOrCreateArray(Elts)); - jl_ppvalue_dillvmt = dbuilder.createPointerType(jl_pvalue_dillvmt,sizeof(jl_value_t**)*8, - __alignof__(jl_value_t**)*8); + jl_ppvalue_dillvmt = dbuilder.createPointerType(jl_pvalue_dillvmt, sizeof(jl_value_t**) * 8, + __alignof__(jl_value_t**) * 8); diargs.push_back(jl_pvalue_dillvmt); // Return Type (ret value) diargs.push_back(jl_pvalue_dillvmt); // First Argument (function) diargs.push_back(jl_ppvalue_dillvmt); // Second Argument (argv) // Third argument (length(argv)) - diargs.push_back(julia_type_to_di((jl_value_t*)jl_int32_type,&dbuilder,false)); + diargs.push_back(_julia_type_to_di(NULL, (jl_value_t*)jl_int32_type, &dbuilder, false)); jl_di_func_sig = dbuilder.createSubroutineType( dbuilder.getOrCreateTypeArray(diargs)); @@ -7178,7 +6781,6 @@ static void init_julia_llvm_env(Module *m) T_ppjlvalue = PointerType::get(T_pjlvalue, 0); T_pprjlvalue = PointerType::get(T_prjlvalue, 0); V_null = Constant::getNullValue(T_pjlvalue); - jl_init_jit(T_pjlvalue); std::vector ftargs(0); ftargs.push_back(T_prjlvalue); // function @@ -7517,11 +7119,6 @@ static void init_julia_llvm_env(Module *m) add_named_global(jl_current_exception_func, &jl_current_exception); #ifdef _OS_WINDOWS_ -#if defined(_CPU_X86_64_) - juliapersonality_func = Function::Create(FunctionType::get(T_int32, true), - Function::ExternalLinkage, "__julia_personality", m); - add_named_global(juliapersonality_func, &__julia_personality); -#endif #ifndef FORCE_ELF #if defined(_CPU_X86_64_) #if defined(_COMPILER_GCC_) @@ -7758,13 +7355,9 @@ static void init_julia_llvm_env(Module *m) false, GlobalVariable::ExternalLinkage, NULL, "jl_world_counter"); add_named_global(jlgetworld_global, &jl_world_counter); - - jl_globalPM = new legacy::PassManager(); - addTargetPasses(jl_globalPM, jl_TargetMachine); - addOptimizationPasses(jl_globalPM, jl_options.opt_level); } -extern "C" void *jl_init_llvm(void) +extern "C" void jl_init_llvm(void) { const char *const argv_tailmerge[] = {"", "-enable-tail-merge=0"}; // NOO TOUCHIE; NO TOUCH! See #922 cl::ParseCommandLineOptions(sizeof(argv_tailmerge)/sizeof(argv_tailmerge[0]), argv_tailmerge, "disable-tail-merge\n"); @@ -7798,11 +7391,6 @@ extern "C" void *jl_init_llvm(void) InitializeNativeTargetAsmParser(); InitializeNativeTargetDisassembler(); - Module *m, *engine_module; - engine_module = new Module("julia", jl_LLVMContext); - m = new Module("julia", jl_LLVMContext); - shadow_output = m; - TargetOptions options = TargetOptions(); //options.PrintMachineCode = true; //Print machine code produced during JIT compiling #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) @@ -7811,23 +7399,6 @@ extern "C" void *jl_init_llvm(void) // to ensure compatibility with GCC codes options.StackAlignmentOverride = 16; #endif - EngineBuilder eb((std::unique_ptr(engine_module))); - std::string ErrorStr; - eb .setEngineKind(EngineKind::JIT) - .setTargetOptions(options) - // Generate simpler code for JIT - .setRelocationModel(Reloc::Static) -#ifdef _P64 - // Make sure we are using the large code model on 64bit - // Let LLVM pick a default suitable for jitting on 32bit - .setCodeModel(CodeModel::Large) -#endif -#ifdef DISABLE_OPT - .setOptLevel(CodeGenOpt::None) -#else - .setOptLevel(jl_options.opt_level == 0 ? CodeGenOpt::None : CodeGenOpt::Aggressive) -#endif - ; Triple TheTriple(sys::getProcessTriple()); #if defined(FORCE_ELF) TheTriple.setObjectFormat(Triple::ELF); @@ -7836,12 +7407,13 @@ extern "C" void *jl_init_llvm(void) auto target = jl_get_llvm_target(imaging_mode, target_flags); auto &TheCPU = target.first; SmallVector targetFeatures(target.second.begin(), target.second.end()); + std::string errorstr; + const Target *TheTarget = TargetRegistry::lookupTarget("", TheTriple, errorstr); + if (!TheTarget) + jl_errorf("%s", errorstr.c_str()); if (jl_processor_print_help || (target_flags & JL_TARGET_UNKNOWN_NAME)) { - std::string errorstr; - const Target *target = TargetRegistry::lookupTarget("", TheTriple, errorstr); - assert(target); std::unique_ptr MSTI( - target->createMCSubtargetInfo(TheTriple.str(), "", "")); + TheTarget->createMCSubtargetInfo(TheTriple.str(), "", "")); if (!MSTI->isCPUStringValid(TheCPU)) jl_errorf("Invalid CPU name %s.", TheCPU.c_str()); if (jl_processor_print_help) { @@ -7851,11 +7423,41 @@ extern "C" void *jl_init_llvm(void) MSTI->setDefaultFeatures("help", ""); } } - jl_TargetMachine = eb.selectTarget( - TheTriple, - "", - TheCPU, - targetFeatures); + // Package up features to be passed to target/subtarget + std::string FeaturesStr; + if (!targetFeatures.empty()) { + SubtargetFeatures Features; + for (unsigned i = 0; i != targetFeatures.size(); ++i) + Features.AddFeature(targetFeatures[i]); + FeaturesStr = Features.getString(); + } + // Allocate a target... + Optional codemodel = +#ifdef _P64 + // Make sure we are using the large code model on 64bit + // Let LLVM pick a default suitable for jitting on 32bit + CodeModel::Large; +#elif JL_LLVM_VERSION < 60000 + CodeModel::JITDefault; +#else + None; +#endif + auto optlevel = +#ifdef DISABLE_OPT + CodeGenOpt::None; +#else + (jl_options.opt_level == 0 ? CodeGenOpt::None : CodeGenOpt::Aggressive); +#endif + jl_TargetMachine = TheTarget->createTargetMachine( + TheTriple.getTriple(), TheCPU, FeaturesStr, + options, + Reloc::Static, // Generate simpler code for JIT + codemodel, + optlevel +#if JL_LLVM_VERSION >= 60000 + , /*JIT*/ true +#endif + ); assert(jl_TargetMachine && "Failed to select target machine -" " Is the LLVM backend for this CPU enabled?"); #if (!defined(_CPU_ARM_) && !defined(_CPU_PPC64_)) @@ -7888,20 +7490,19 @@ extern "C" void *jl_init_llvm(void) #endif #ifdef JL_USE_PERF_JITEVENTS - if (jl_using_perf_jitevents) { + if (jl_using_perf_jitevents) jl_ExecutionEngine->RegisterJITEventListener(JITEventListener::createPerfJITEventListener()); - } #endif - - // Now that the execution engine exists, initialize all modules - jl_setup_module(engine_module); - jl_setup_module(m); - return (void*)m; } extern "C" void jl_init_codegen(void) { - Module *m = (Module *)jl_init_llvm(); + jl_init_llvm(); + // Now that the execution engine exists, initialize all modules + jl_init_jit(); + Module *m = new Module("julia", jl_LLVMContext); + shadow_output = m; + jl_setup_module(m); init_julia_llvm_env(m); SBOX_F_PERM(int8,int8); UBOX_F_PERM(uint8,uint8); @@ -7915,7 +7516,8 @@ extern "C" void jl_init_codegen(void) jl_init_intrinsic_functions_codegen(m); } -// for debugging from gdb +// the rest of this file are convenience functions +// that are exported for assisting with debugging from gdb extern "C" void jl_dump_llvm_value(void *v) { llvm_dump((Value*)v); diff --git a/src/datatype.c b/src/datatype.c index 779d0a30c98af..1016d189cf8d8 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -551,8 +551,6 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( t->mutabl = mutabl; t->ninitialized = ninitialized; t->instance = NULL; - t->struct_decl = NULL; - t->ditype = NULL; t->size = 0; t->name = NULL; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index d1f1846d98a31..9d83f4d4765e2 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -73,7 +73,7 @@ struct ObjectInfo { // Maintain a mapping of unrealized function names -> linfo objects // so that when we see it get emitted, we can add a link back to the linfo // that it came from (providing name, type signature, file info, etc.) -static StringMap ncode_in_flight; +static StringMap codeinst_in_flight; static std::string mangle(const std::string &Name, const DataLayout &DL) { std::string MangledName; @@ -85,7 +85,7 @@ static std::string mangle(const std::string &Name, const DataLayout &DL) } void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL) { - ncode_in_flight[mangle(name, DL)] = codeinst; + codeinst_in_flight[mangle(name, DL)] = codeinst; } @@ -195,10 +195,10 @@ class JuliaJITEventListener: public JITEventListener auto SavedObject = L.getObjectForDebug(Object).takeBinary(); // If the debug object is unavailable, save (a copy of) the original object - // for our backtraces + // for our backtraces. + // This copy seems unfortunate, but there doesn't seem to be a way to take + // ownership of the original buffer. if (!SavedObject.first) { - // This is unfortunate, but there doesn't seem to be a way to take - // ownership of the original buffer auto NewBuffer = MemoryBuffer::getMemBufferCopy( Object.getData(), Object.getFileName()); auto NewObj = object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef()); @@ -399,11 +399,11 @@ class JuliaJITEventListener: public JITEventListener (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); #endif - StringMap::iterator linfo_it = ncode_in_flight.find(sName); + StringMap::iterator codeinst_it = codeinst_in_flight.find(sName); jl_code_instance_t *codeinst = NULL; - if (linfo_it != ncode_in_flight.end()) { - codeinst = linfo_it->second; - ncode_in_flight.erase(linfo_it); + if (codeinst_it != codeinst_in_flight.end()) { + codeinst = codeinst_it->second; + codeinst_in_flight.erase(codeinst_it); } uv_rwlock_wrlock(&threadsafe); if (codeinst) diff --git a/src/disasm.cpp b/src/disasm.cpp index bff5b0c7938ab..f4c5d661f6901 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -375,10 +375,66 @@ void LineNumberAnnotatedWriter::emitBasicBlockEndAnnot( LinePrinter.emit_finish(Out); } +static void jl_strip_llvm_debug(Module *m, bool all_meta, LineNumberAnnotatedWriter *AAW) +{ + // strip metadata from all instructions in all functions in the module + Instruction *deletelast = nullptr; // can't actually delete until the iterator advances + for (Function &f : m->functions()) { + if (AAW) + AAW->addSubprogram(&f, f.getSubprogram()); + for (BasicBlock &f_bb : f) { + for (Instruction &inst : f_bb) { + if (deletelast) { + deletelast->eraseFromParent(); + deletelast = nullptr; + } + // remove dbg.declare and dbg.value calls + if (isa(inst) || isa(inst)) { + deletelast = &inst; + continue; + } + + // iterate over all metadata kinds and set to NULL to remove + if (all_meta) { + SmallVector, 4> MDForInst; + inst.getAllMetadataOtherThanDebugLoc(MDForInst); + for (const auto &md_iter : MDForInst) { + inst.setMetadata(md_iter.first, NULL); + } + } + // record debug location before erasing it + if (AAW) + AAW->addDebugLoc(&inst, inst.getDebugLoc()); + inst.setDebugLoc(DebugLoc()); + } + if (deletelast) { + deletelast->eraseFromParent(); + deletelast = nullptr; + } + } + f.setSubprogram(NULL); + } + if (all_meta) { + for (GlobalObject &g : m->global_objects()) { + g.clearMetadata(); + } + } + // now that the subprogram is not referenced, we can delete it too + if (NamedMDNode *md = m->getNamedMetadata("llvm.dbg.cu")) + m->eraseNamedMetadata(md); + //if (NamedMDNode *md = m->getNamedMetadata("llvm.module.flags")) + // m->eraseNamedMetadata(md); +} + +void jl_strip_llvm_debug(Module *m) +{ + jl_strip_llvm_debug(m, false, NULL); +} + // print an llvm IR acquired from jl_get_llvmf // warning: this takes ownership of, and destroys, f->getParent() extern "C" JL_DLLEXPORT -jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_module, const char *debuginfo) +jl_value_t *jl_dump_function_ir(void *f, char strip_ir_metadata, char dump_module, const char *debuginfo) { std::string code; llvm::raw_string_ostream stream(code); @@ -396,43 +452,8 @@ jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_modul } else { Module *m = llvmf->getParent(); - if (strip_ir_metadata) { - // strip metadata from all instructions in all functions in the module - Instruction *deletelast = nullptr; // can't actually delete until the iterator advances - for (Function &f2 : m->functions()) { - AAW.addSubprogram(&f2, f2.getSubprogram()); - for (BasicBlock &f2_bb : f2) { - for (Instruction &inst : f2_bb) { - if (deletelast) { - deletelast->eraseFromParent(); - deletelast = nullptr; - } - // remove dbg.declare and dbg.value calls - if (isa(inst) || isa(inst)) { - deletelast = &inst; - continue; - } - - // iterate over all metadata kinds and set to NULL to remove - SmallVector, 4> MDForInst; - inst.getAllMetadataOtherThanDebugLoc(MDForInst); - for (const auto &md_iter : MDForInst) { - inst.setMetadata(md_iter.first, NULL); - } - // record debug location before erasing it - AAW.addDebugLoc(&inst, inst.getDebugLoc()); - inst.setDebugLoc(DebugLoc()); - } - if (deletelast) { - deletelast->eraseFromParent(); - deletelast = nullptr; - } - } - } - for (GlobalObject &g2 : m->global_objects()) { - g2.clearMetadata(); - } - } + if (strip_ir_metadata) + jl_strip_llvm_debug(m, true, &AAW); if (dump_module) { m->print(stream, &AAW); } @@ -629,8 +650,9 @@ void SymbolTable::createSymbols() } else { const char *global = lookupLocalPC(addr); - if (global) + if (global && global[0]) isymb->second = global; + // TODO: free(global)? } } } @@ -741,7 +763,6 @@ static void jl_dump_asm_internal( const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple.str(), err); // Set up required helpers and streamer - std::unique_ptr Streamer; SourceMgr SrcMgr; MCTargetOptions Options; @@ -774,17 +795,18 @@ static void jl_dump_asm_internal( } bool ShowEncoding = false; - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr - MCIA(TheTarget->createMCInstrAnalysis(MCII.get())); - MCInstPrinter *IP = - TheTarget->createMCInstPrinter(TheTriple, OutputAsmVariant, *MAI, *MCII, *MRI); + std::unique_ptr MCII( + TheTarget->createMCInstrInfo()); + std::unique_ptr MCIA( + TheTarget->createMCInstrAnalysis(MCII.get())); + std::unique_ptr IP( + TheTarget->createMCInstPrinter(TheTriple, OutputAsmVariant, *MAI, *MCII, *MRI)); //IP->setPrintImmHex(true); // prefer hex or decimal immediates - std::unique_ptr CE = 0; - std::unique_ptr MAB = 0; + std::unique_ptr CE; + std::unique_ptr MAB; if (ShowEncoding) { - CE = std::unique_ptr(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MAB = std::unique_ptr(TheTarget->createMCAsmBackend(*STI, *MRI, Options)); + CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); + MAB.reset(TheTarget->createMCAsmBackend(*STI, *MRI, Options)); } // createAsmStreamer expects a unique_ptr to a formatted stream, which means @@ -792,9 +814,12 @@ static void jl_dump_asm_internal( // start out with a raw stream, and create formatted stream from it here. // LLVM will destroy the formatted stream, and we keep the raw stream. std::unique_ptr ustream(new formatted_raw_ostream(rstream)); - Streamer.reset(TheTarget->createAsmStreamer(Ctx, std::move(ustream), /*asmverbose*/true, - /*useDwarfDirectory*/ true, - IP, std::move(CE), std::move(MAB), /*ShowInst*/ false)); + std::unique_ptr Streamer( + TheTarget->createAsmStreamer(Ctx, std::move(ustream), /*asmverbose*/true, + /*useDwarfDirectory*/ true, + IP.release(), + std::move(CE), std::move(MAB), + /*ShowInst*/ false)); Streamer->InitSections(true); // Make the MemoryObject wrapper diff --git a/src/dump.c b/src/dump.c index 27ebab33fc8ce..b9606cf8a7ee5 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1432,9 +1432,7 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v assert(pos == backref_list.len - 1 && "nothing should have been deserialized since assigning pos"); backref_list.items[pos] = dt; dt->size = size; - dt->struct_decl = NULL; dt->instance = NULL; - dt->ditype = NULL; dt->abstract = flags & 1; dt->mutabl = (flags >> 1) & 1; int has_layout = (flags >> 2) & 1; diff --git a/src/gf.c b/src/gf.c index 4bc30ad13a46a..e7c3cae97832c 100644 --- a/src/gf.c +++ b/src/gf.c @@ -187,6 +187,8 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t *mi, size_t world, int force) #ifdef ENABLE_INFERENCE if (mi->inInference && !force) return NULL; + if (jl_is_method(mi->def.method) && mi->def.method->unspecialized == mi) + return NULL; // be careful never to infer the unspecialized method, this would not be valid jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 3); @@ -252,11 +254,11 @@ JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *mi, size_t mi if (codeinst->min_world <= min_world && max_world <= codeinst->max_world) { jl_value_t *code = codeinst->inferred; if (code && (code == jl_nothing || jl_ast_flag_inferred((jl_array_t*)code))) - return (jl_value_t *)codeinst; + return (jl_value_t*)codeinst; } codeinst = codeinst->next; } - return (jl_value_t *)jl_nothing; + return (jl_value_t*)jl_nothing; } @@ -292,8 +294,6 @@ JL_DLLEXPORT jl_code_instance_t *jl_set_method_inferred( codeinst->def = mi; codeinst->min_world = min_world; codeinst->max_world = max_world; - codeinst->functionObjectsDecls.functionObject = NULL; - codeinst->functionObjectsDecls.specFunctionObject = NULL; codeinst->rettype = rettype; codeinst->inferred = inferred; //codeinst->edges = NULL; @@ -306,6 +306,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_set_method_inferred( jl_atomic_store_release(&codeinst->invoke, jl_fptr_const_return); } codeinst->specptr.fptr = NULL; + codeinst->isspecsig = 0; if (jl_is_method(mi->def.method)) JL_LOCK(&mi->def.method->writelock); codeinst->next = mi->cache; @@ -1832,7 +1833,8 @@ jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method JL_PROPA { // one unspecialized version of a function can be shared among all cached specializations jl_method_t *def = method->def.method; - if (def->source == NULL) { + if (!jl_is_method(def) || def->source == NULL) { + // generated functions might instead randomly just never get inferred, sorry return method; } if (def->unspecialized == NULL) { @@ -1846,7 +1848,8 @@ jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method JL_PROPA return def->unspecialized; } -jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) + +jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) { jl_code_instance_t *codeinst; codeinst = mi->cache; @@ -1857,6 +1860,14 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t } codeinst = codeinst->next; } + return NULL; +} + +jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) +{ + jl_code_instance_t *codeinst = jl_method_compiled(mi, world); + if (codeinst) + return codeinst; if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) { @@ -1867,6 +1878,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t if (unspec && unspec->invoke != NULL) { jl_code_instance_t *codeinst = jl_set_method_inferred(mi, (jl_value_t*)jl_any_type, NULL, NULL, 0, 1, ~(size_t)0); + codeinst->isspecsig = 0; codeinst->specptr = unspec->specptr; codeinst->rettype_const = unspec->rettype_const; jl_atomic_store_release(&codeinst->invoke, unspec->invoke); @@ -1887,46 +1899,24 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t } } - JL_LOCK(&codegen_lock); - codeinst = mi->cache; - while (codeinst) { - if (codeinst->min_world <= world && world <= codeinst->max_world && codeinst->functionObjectsDecls.functionObject != NULL) - break; - codeinst = codeinst->next; - } - if (codeinst == NULL) { - // if we don't have any decls already, try to generate it now - jl_code_info_t *src = NULL; - if (jl_is_method(mi->def.method) && jl_rettype_inferred(mi, world, world) == jl_nothing && - jl_symbol_name(mi->def.method->name)[0] != '@') { - // don't bother with typeinf on macros or toplevel thunks - // but try to infer everything else - src = jl_type_infer(mi, world, 0); - } - codeinst = jl_compile_linfo(mi, src, world, &jl_default_cgparams); - if (!codeinst) { - jl_method_instance_t *unspec = jl_get_unspecialized(mi); - jl_code_instance_t *ucache = jl_get_method_inferred(unspec, (jl_value_t*)jl_any_type, 1, ~(size_t)0); - // ask codegen to make the fptr for unspec - if (ucache->invoke == NULL) - jl_generate_fptr(ucache); - if (ucache->invoke != jl_fptr_sparam && - ucache->invoke != jl_fptr_interpret_call) { - JL_UNLOCK(&codegen_lock); - return ucache; - } - jl_code_instance_t *codeinst = jl_set_method_inferred(mi, (jl_value_t*)jl_any_type, NULL, NULL, - 0, 1, ~(size_t)0); - codeinst->specptr = ucache->specptr; - codeinst->rettype_const = ucache->rettype_const; - jl_atomic_store_release(&codeinst->invoke, ucache->invoke); - JL_UNLOCK(&codegen_lock); - return codeinst; + codeinst = jl_generate_fptr(mi, world); + if (!codeinst) { + jl_method_instance_t *unspec = jl_get_unspecialized(mi); + jl_code_instance_t *ucache = jl_get_method_inferred(unspec, (jl_value_t*)jl_any_type, 1, ~(size_t)0); + // ask codegen to make the fptr for unspec + if (ucache->invoke == NULL) + jl_generate_fptr_for_unspecialized(ucache); + if (ucache->invoke != jl_fptr_sparam && + ucache->invoke != jl_fptr_interpret_call) { + return ucache; } + codeinst = jl_set_method_inferred(mi, (jl_value_t*)jl_any_type, NULL, NULL, + 0, 1, ~(size_t)0); + codeinst->isspecsig = 0; + codeinst->specptr = ucache->specptr; + codeinst->rettype_const = ucache->rettype_const; + jl_atomic_store_release(&codeinst->invoke, ucache->invoke); } - - JL_UNLOCK(&codegen_lock); - jl_generate_fptr(codeinst); return codeinst; } @@ -2018,11 +2008,12 @@ jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES static void _generate_from_hint(jl_method_instance_t *mi, size_t world) { int generating_llvm = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc; - jl_code_info_t *src = NULL; // If we are saving ji files (e.g. package pre-compilation or intermediate sysimg build steps), // don't bother generating anything since it won't be saved. if (jl_rettype_inferred(mi, world, world) == jl_nothing) - src = jl_type_infer(mi, world, 1); + (void)jl_type_infer(mi, world, 1); + // If we are saving ji files (e.g. package pre-compilation or intermediate sysimg build steps), + // don't bother generating output in the current environment if (generating_llvm) { jl_value_t *codeinst = jl_rettype_inferred(mi, world, world); if (codeinst != jl_nothing) @@ -2030,7 +2021,7 @@ static void _generate_from_hint(jl_method_instance_t *mi, size_t world) return; // probably not a good idea to generate code // If we are saving LLVM or native code, generate the LLVM IR so that it'll // be included in the saved LLVM module. - (void)jl_compile_linfo(mi, src, world, &jl_default_cgparams); + // TODO: compilation is now stateless } } @@ -2148,13 +2139,13 @@ static void show_call(jl_value_t *F, jl_value_t **args, uint32_t nargs) STATIC_INLINE jl_value_t *verify_type(jl_value_t *v) JL_NOTSAFEPOINT { - assert(jl_typeof(jl_typeof(v))); + assert(v && jl_typeof(v) && jl_typeof(jl_typeof(v)) == (jl_value_t*)jl_datatype_type); return v; } STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc, size_t world) { - // manually inline key parts of jl_compile_method_internal: + // manually inlined copy of jl_method_compiled jl_code_instance_t *codeinst = mfunc->cache; while (codeinst) { if (codeinst->min_world <= world && world <= codeinst->max_world && codeinst->invoke != NULL) { diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index deeca508061c6..9056cb5fba10d 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -147,7 +147,7 @@ static Value *uint_cnvt(jl_codectx_t &ctx, Type *to, Value *x) return ctx.builder.CreateZExt(x, to); } -static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) +static Constant *julia_const_to_llvm(jl_codectx_t &ctx, const void *ptr, jl_datatype_t *bt) { // assumes `jl_is_pointerfree(bt)`. // `ptr` can point to a inline field, do not read the tag from it. @@ -156,7 +156,7 @@ static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) if (bt == jl_bool_type) return ConstantInt::get(T_int8, (*(const uint8_t*)ptr) ? 1 : 0); - Type *lt = julia_struct_to_llvm((jl_value_t*)bt, NULL, NULL); + Type *lt = julia_struct_to_llvm(ctx, (jl_value_t*)bt, NULL, NULL); if (jl_is_vecelement_type((jl_value_t*)bt) && !jl_is_uniontype(jl_tparam0(bt))) bt = (jl_datatype_t*)jl_tparam0(bt); @@ -194,7 +194,7 @@ static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) for (size_t i = 0; i < nf; i++) { size_t offs = jl_field_offset(bt, i); jl_value_t *ft = jl_field_type(bt, i); - Type *lft = julia_type_to_llvm(ft); + Type *lft = julia_type_to_llvm(ctx, ft); if (type_is_ghost(lft)) continue; assert(!jl_field_isptr(bt, i)); @@ -250,7 +250,7 @@ static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) fields.push_back(ConstantInt::get(T_int8, sel)); } else { - Constant *val = julia_const_to_llvm(ov, (jl_datatype_t*)ft); + Constant *val = julia_const_to_llvm(ctx, ov, (jl_datatype_t*)ft); fields.push_back(val); } } @@ -263,7 +263,7 @@ static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) return ConstantArray::get(at, fields); } -static Constant *julia_const_to_llvm(jl_value_t *e) +static Constant *julia_const_to_llvm(jl_codectx_t &ctx, jl_value_t *e) { if (e == jl_true) return ConstantInt::get(T_int8, 1); @@ -272,7 +272,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e) jl_value_t *bt = jl_typeof(e); if (!jl_is_pointerfree(bt)) return NULL; - return julia_const_to_llvm(e, (jl_datatype_t*)bt); + return julia_const_to_llvm(ctx, e, (jl_datatype_t*)bt); } static jl_cgval_t ghostValue(jl_value_t *ty); @@ -333,7 +333,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va return UndefValue::get(to); // type mismatch error } - Constant *c = x.constant ? julia_const_to_llvm(x.constant) : NULL; + Constant *c = x.constant ? julia_const_to_llvm(ctx, x.constant) : NULL; if (!x.ispointer() || c) { // already unboxed, but sometimes need conversion Value *unboxed = c ? c : x.V; if (!dest) @@ -436,7 +436,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) // Examine the second argument // bool isboxed; - Type *vxt = julia_type_to_llvm(v.typ, &isboxed); + Type *vxt = julia_type_to_llvm(ctx, v.typ, &isboxed); if (!jl_is_primitivetype(v.typ) || jl_datatype_size(v.typ) != nb) { Value *typ = emit_typeof_boxed(ctx, v); @@ -469,7 +469,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) if (!v.ispointer()) vx = v.V; else if (v.constant) - vx = julia_const_to_llvm(v.constant); + vx = julia_const_to_llvm(ctx, v.constant); if (v.ispointer() && vx == NULL) { // try to load as original Type, to preserve llvm optimizations @@ -603,7 +603,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) } else { bool isboxed; - Type *ptrty = julia_type_to_llvm(ety, &isboxed); + Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed); assert(!isboxed); if (!type_is_ghost(ptrty)) { Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); @@ -671,7 +671,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) } else { bool isboxed; - Type *ptrty = julia_type_to_llvm(ety, &isboxed); + Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed); assert(!isboxed); if (!type_is_ghost(ptrty)) { thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); @@ -766,7 +766,7 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ Value *ifelse_result; bool isboxed = t1 != t2 || !deserves_stack(t1); - Type *llt1 = isboxed ? T_prjlvalue : julia_type_to_llvm(t1); + Type *llt1 = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, t1); if (!isboxed) { if (type_is_ghost(llt1)) return x; @@ -1092,7 +1092,7 @@ static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, Value **arg *newtyp = tuptyp; Value *tupval; - tupval = UndefValue::get(julia_type_to_llvm((jl_value_t*)tuptyp)); + tupval = UndefValue::get(julia_type_to_llvm(ctx, (jl_value_t*)tuptyp)); tupval = ctx.builder.CreateInsertValue(tupval, val, ArrayRef(0)); tupval = ctx.builder.CreateInsertValue(tupval, obyte, ArrayRef(1)); return tupval; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index b3882e656413e..98d5514dee251 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,263 +1,342 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license +// Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). + #include "llvm-version.h" #include "platform.h" #include "options.h" -#if defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_) -# define JL_DISABLE_FPO -#endif #include #include -// analysis passes -#include -#include -#include -#include -#include -#include -#include -#if defined(USE_POLLY) -#include -#include -#include -#if defined(USE_POLLY_ACC) -#include -#endif -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if JL_LLVM_VERSION >= 100000 -#include -#endif - -namespace llvm { - extern Pass *createLowerSimdLoopPass(); -} - -#include -#include - -#include -#include #include -#include "llvm/Object/ArchiveWriter.h" - -// target support -#include -#include -#include #include - +#include #include #include #include #include #include -#include "codegen_shared.h" using namespace llvm; #include "julia.h" #include "julia_internal.h" +#include "codegen_shared.h" #include "jitlayers.h" #include "julia_assert.h" -#if JL_LLVM_VERSION < 100000 -static const TargetMachine::CodeGenFileType CGFT_ObjectFile = TargetMachine::CGFT_ObjectFile; -#endif - RTDyldMemoryManager* createRTDyldMemoryManager(void); -static IntegerType *T_uint32; -static IntegerType *T_uint64; -static IntegerType *T_size; -static Type *T_psize; -static Type *T_pjlvalue; -void jl_init_jit(Type *T_pjlvalue_) +void jl_init_jit(void) { } + +// Snooping on which functions are being compiled, and how long it takes +JL_STREAM *dump_compiles_stream = NULL; +extern "C" JL_DLLEXPORT +void jl_dump_compiles(void *s) { - T_uint32 = Type::getInt32Ty(jl_LLVMContext); - T_uint64 = Type::getInt64Ty(jl_LLVMContext); - if (sizeof(size_t) == 8) - T_size = T_uint64; - else - T_size = T_uint32; - T_psize = PointerType::get(T_size, 0); - T_pjlvalue = T_pjlvalue_; + dump_compiles_stream = (JL_STREAM*)s; } -// Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). +static void jl_add_to_ee(); +static uint64_t getAddressForFunction(StringRef fname); -void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM) +void jl_jit_globals(std::map &globals) { - PM->add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); - PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); + for (auto &global : globals) { + Constant *P = literal_static_pointer_val(global.first, global.second->getValueType()); + global.second->setInitializer(P); + global.second->setConstant(true); + global.second->setLinkage(GlobalValue::PrivateLinkage); + global.second->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + } } -// this defines the set of optimization passes defined for Julia at various optimization levels. -// it assumes that the TLI and TTI wrapper passes have already been added. -void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, - bool lower_intrinsics, bool dump_native) -{ -#ifdef JL_DEBUG_BUILD - PM->add(createGCInvariantVerifierPass(true)); - PM->add(createVerifierPass()); -#endif +// this generates llvm code for the lambda info +// and adds the result to the jitlayers +// (and the shadow module), +// and generates code for it +static jl_callptr_t _jl_compile_codeinst( + jl_code_instance_t *codeinst, + jl_code_info_t *src, + size_t world) +{ + // TODO: Merge with jl_dump_compiles? + static ios_t f_precompile; + static JL_STREAM* s_precompile = NULL; + + // caller must hold codegen_lock + // and have disabled finalizers + JL_TIMING(CODEGEN); + uint64_t start_time = 0; + if (dump_compiles_stream != NULL) + start_time = jl_hrtime(); + + assert(jl_is_code_instance(codeinst)); + assert(codeinst->min_world <= world && (codeinst->max_world >= world || codeinst->max_world == 0) && + "invalid world for method-instance"); + assert(src && jl_is_code_info(src)); + + jl_callptr_t fptr = NULL; + // emit the code in LLVM IR form + jl_codegen_params_t params; + params.cache = true; + params.world = world; + std::map emitted; + jl_compile_result_t result = jl_emit_codeinst(codeinst, src, params); + if (std::get<0>(result)) + emitted[codeinst] = std::move(result); + jl_compile_workqueue(emitted, params); + + jl_jit_globals(params.globals); + for (auto &def : emitted) { + // Add the results to the execution engine now + jl_finalize_module(std::move(std::get<0>(def.second))); + } + jl_add_to_ee(); + for (auto &def : emitted) { + jl_code_instance_t *this_code = def.first; + jl_llvm_functions_t decls = std::get<1>(def.second); + jl_callptr_t addr; + bool isspecsig = false; + if (decls.functionObject == "jl_fptr_args") { + addr = &jl_fptr_args; + } + else if (decls.functionObject == "jl_fptr_sparam") { + addr = &jl_fptr_sparam; + } + else { + addr = (jl_callptr_t)getAddressForFunction(decls.functionObject); + isspecsig = true; + } + if (this_code->invoke == NULL) { + // once set, don't change invoke-ptr, as that leads to race conditions + // with the (not) simultaneous updates to invoke and specptr + if (!decls.specFunctionObject.empty()) { + this_code->specptr.fptr = (void*)getAddressForFunction(decls.specFunctionObject); + this_code->isspecsig = isspecsig; + } + this_code->invoke = addr; + } + else if (this_code->invoke == jl_fptr_const_return && !decls.specFunctionObject.empty()) { + // hack to export this pointer value to jl_dump_method_asm + this_code->specptr.fptr = (void*)getAddressForFunction(decls.specFunctionObject); + } + if (this_code== codeinst) + fptr = addr; + } -#if defined(JL_ASAN_ENABLED) - PM->add(createAddressSanitizerFunctionPass()); -#endif -#if defined(JL_MSAN_ENABLED) - PM->add(llvm::createMemorySanitizerPass(true)); -#endif - if (opt_level < 2) { - PM->add(createCFGSimplificationPass()); // Clean up disgusting code - if (opt_level == 1) { - PM->add(createSROAPass()); // Break up aggregate allocas - PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. - PM->add(createEarlyCSEPass()); + uint64_t end_time = 0; + if (dump_compiles_stream != NULL) + end_time = jl_hrtime(); + + // If logging of the compilation stream is enabled, + // then dump the method-instance specialization type to the stream + jl_method_instance_t *mi = codeinst->def; + if (jl_is_method(mi->def.method)) { + if (jl_options.trace_compile != NULL) { + if (s_precompile == NULL) { + const char* t = jl_options.trace_compile; + if (!strncmp(t, "stderr", 6)) + s_precompile = JL_STDERR; + else { + if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL) + jl_errorf("cannot open precompile statement file \"%s\" for writing", t); + s_precompile = (JL_STREAM*) &f_precompile; + } + } + if (!jl_has_free_typevars(mi->specTypes)) { + jl_printf(s_precompile, "precompile("); + jl_static_show(s_precompile, mi->specTypes); + jl_printf(s_precompile, ")\n"); + + if (s_precompile != JL_STDERR) + ios_flush(&f_precompile); + } } - PM->add(createMemCpyOptPass()); // Remove memcpy / form memset - PM->add(createAlwaysInlinerLegacyPass()); // Respect always_inline - if (lower_intrinsics) { - PM->add(createBarrierNoopPass()); - PM->add(createLowerExcHandlersPass()); - PM->add(createGCInvariantVerifierPass(false)); - PM->add(createLateLowerGCFramePass()); - PM->add(createFinalLowerGCPass()); - PM->add(createLowerPTLSPass(dump_native)); + if (dump_compiles_stream != NULL) { + jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", end_time - start_time); + jl_static_show(dump_compiles_stream, mi->specTypes); + jl_printf(dump_compiles_stream, "\"\n"); } - PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop - if (dump_native) - PM->add(createMultiVersioningPass()); - return; } - PM->add(createPropagateJuliaAddrspaces()); - PM->add(createScopedNoAliasAAWrapperPass()); - PM->add(createTypeBasedAAWrapperPass()); - if (opt_level >= 3) { - PM->add(createBasicAAWrapperPass()); + return fptr; +} + +// get the address of a C-callable entry point for a function +extern "C" JL_DLLEXPORT +void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) +{ + JL_GC_PUSH1(&argt); + JL_LOCK(&codegen_lock); + jl_codegen_params_t params; + Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt, params); + jl_jit_globals(params.globals); + assert(params.workqueue.empty()); + jl_add_to_ee(); + JL_GC_POP(); + void *ptr = (void*)getAddressForFunction(llvmf->getName()); + JL_UNLOCK(&codegen_lock); + return ptr; +} + +// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name +extern "C" JL_DLLEXPORT +void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) +{ + JL_LOCK(&codegen_lock); + jl_codegen_params_t params; + Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt, params); + jl_jit_globals(params.globals); + assert(params.workqueue.empty()); + // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) + jl_add_to_ee(); + uint64_t Addr = getAddressForFunction(llvmf->getName()); + + if (imaging_mode) + llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); + + // make the alias to the shadow_module + GlobalAlias *GA = + GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); + + // make sure the alias name is valid for the current session + jl_ExecutionEngine->addGlobalMapping(GA, (void*)(uintptr_t)Addr); + JL_UNLOCK(&codegen_lock); +} + +// this compiles li and emits fptr +extern "C" +jl_code_instance_t *jl_generate_fptr(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) +{ + JL_LOCK(&codegen_lock); // also disables finalizers, to prevent any unexpected recursion + // if we don't have any decls already, try to generate it now + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + jl_value_t *ci = jl_rettype_inferred(mi, world, world); + jl_code_instance_t *codeinst = (ci == jl_nothing ? NULL : (jl_code_instance_t*)ci); + if (codeinst) { + src = (jl_code_info_t*)codeinst->inferred; + if ((jl_value_t*)src == jl_nothing) + src = NULL; + else if (jl_is_method(mi->def.method)) + src = jl_uncompress_ast(mi->def.method, codeinst, (jl_array_t*)src); } - // list of passes from vmkit - PM->add(createCFGSimplificationPass()); // Clean up disgusting code - PM->add(createDeadCodeEliminationPass()); - PM->add(createSROAPass()); // Kill useless allocas - - PM->add(createMemCpyOptPass()); - - PM->add(createAlwaysInlinerLegacyPass()); // Respect always_inline - - // Running `memcpyopt` between this and `sroa` seems to give `sroa` a hard time - // merging the `alloca` for the unboxed data and the `alloca` created by the `alloc_opt` - // pass. - PM->add(createAllocOptPass()); - PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. - // Now that SROA has cleaned up for front-end mess, a lot of control flow should - // be more evident - try to clean it up. - PM->add(createCFGSimplificationPass()); // Merge & remove BBs - if (dump_native) - PM->add(createMultiVersioningPass()); - PM->add(createSROAPass()); // Break up aggregate allocas - PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. - PM->add(createJumpThreadingPass()); // Thread jumps. - PM->add(createInstructionCombiningPass()); // Combine silly seq's - - //PM->add(createCFGSimplificationPass()); // Merge & remove BBs - PM->add(createReassociatePass()); // Reassociate expressions - - // this has the potential to make some things a bit slower - //PM->add(createBBVectorizePass()); - - PM->add(createEarlyCSEPass()); //// **** - - // Load forwarding above can expose allocations that aren't actually used - // remove those before optimizing loops. - PM->add(createAllocOptPass()); - PM->add(createLoopIdiomPass()); //// **** - PM->add(createLoopRotatePass()); // Rotate loops. -#ifdef USE_POLLY - // LCSSA (which has already run at this point due to the dependencies of the - // above passes) introduces redundant phis that hinder Polly. Therefore we - // run InstCombine here to remove them. - PM->add(createInstructionCombiningPass()); - PM->add(polly::createCodePreparationPass()); - polly::registerPollyPasses(*PM); - PM->add(polly::createCodegenCleanupPass()); -#endif - // LoopRotate strips metadata from terminator, so run LowerSIMD afterwards - PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop - PM->add(createLICMPass()); // Hoist loop invariants - PM->add(createLoopUnswitchPass()); // Unswitch loops. - // Subsequent passes not stripping metadata from terminator - PM->add(createInstructionCombiningPass()); - PM->add(createIndVarSimplifyPass()); // Canonicalize indvars - PM->add(createLoopDeletionPass()); // Delete dead loops - PM->add(createSimpleLoopUnrollPass()); // Unroll small loops - //PM->add(createLoopStrengthReducePass()); // (jwb added) - - // Run our own SROA on heap objects before LLVM's - PM->add(createAllocOptPass()); - // Re-run SROA after loop-unrolling (useful for small loops that operate, - // over the structure of an aggregate) - PM->add(createSROAPass()); // Break up aggregate allocas - PM->add(createInstructionCombiningPass()); // Clean up after the unroller - PM->add(createGVNPass()); // Remove redundancies - PM->add(createMemCpyOptPass()); // Remove memcpy / form memset - PM->add(createSCCPPass()); // Constant prop with SCCP - - // Run instcombine after redundancy elimination to exploit opportunities - // opened up by them. - PM->add(createInstructionCombiningPass()); - PM->add(createJumpThreadingPass()); // Thread jumps - PM->add(createDeadStoreEliminationPass()); // Delete dead stores - - // More dead allocation (store) deletion before loop optimization - PM->add(createAllocOptPass()); - // see if all of the constant folding has exposed more loops - // to simplification and deletion - // this helps significantly with cleaning up iteration - PM->add(createCFGSimplificationPass()); // Merge & remove BBs - PM->add(createLoopIdiomPass()); - PM->add(createLoopDeletionPass()); // Delete dead loops - PM->add(createJumpThreadingPass()); // Thread jumps - PM->add(createSLPVectorizerPass()); // Vectorize straight-line code - PM->add(createAggressiveDCEPass()); // Delete dead instructions - PM->add(createInstructionCombiningPass()); // Clean up after SLP loop vectorizer - PM->add(createLoopVectorizePass()); // Vectorize loops - PM->add(createInstructionCombiningPass()); // Clean up after loop vectorizer - - if (lower_intrinsics) { - // LowerPTLS removes an indirect call. As a result, it is likely to trigger - // LLVM's devirtualization heuristics, which would result in the entire - // pass pipeline being re-exectuted. Prevent this by inserting a barrier. - PM->add(createBarrierNoopPass()); - PM->add(createLowerExcHandlersPass()); - PM->add(createGCInvariantVerifierPass(false)); - PM->add(createLateLowerGCFramePass()); - PM->add(createFinalLowerGCPass()); - // Remove dead use of ptls - PM->add(createDeadCodeEliminationPass()); - PM->add(createLowerPTLSPass(dump_native)); - // Clean up write barrier and ptls lowering - PM->add(createCFGSimplificationPass()); + if (src == NULL && jl_is_method(mi->def.method) && + jl_symbol_name(mi->def.method->name)[0] != '@') { + // If the caller didn't provide the source, + // see if it is inferred, or try to infer it for ourself. + // (but don't bother with typeinf on macros or toplevel thunks) + src = jl_type_infer(mi, world, 0); + } + jl_code_instance_t *compiled = jl_method_compiled(mi, world); + if (compiled) { + codeinst = compiled; + } + else if (src && jl_is_code_info(src)) { + if (!codeinst) { + codeinst = jl_get_method_inferred(mi, src->rettype, src->min_world, src->max_world); + if (src->inferred && !codeinst->inferred) + codeinst->inferred = jl_nothing; + } + _jl_compile_codeinst(codeinst, src, world); + if (codeinst->invoke == NULL) + codeinst = NULL; + } + else { + codeinst = NULL; } - PM->add(createCombineMulAddPass()); + JL_UNLOCK(&codegen_lock); + JL_GC_POP(); + return codeinst; } +extern "C" +void jl_generate_fptr_for_unspecialized(jl_code_instance_t *unspec) +{ + if (unspec->invoke != NULL) + return; + JL_LOCK(&codegen_lock); + if (unspec->invoke == NULL) { + jl_code_info_t *src = NULL; + JL_GC_PUSH1(&src); + jl_method_t *def = unspec->def->def.method; + if (jl_is_method(def)) { + src = (jl_code_info_t*)def->source; + if (src == NULL) { + // TODO: this is wrong + assert(def->generator); + // TODO: jl_code_for_staged can throw + src = jl_code_for_staged(unspec->def); + } + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(def, NULL, (jl_array_t*)src); + } + else { + src = (jl_code_info_t*)unspec->def->uninferred; + } + assert(src && jl_is_code_info(src)); + _jl_compile_codeinst(unspec, src, unspec->min_world); + if (unspec->invoke == NULL) + // if we hit a codegen bug (or ran into a broken generated function or llvmcall), fall back to the interpreter as a last resort + unspec->invoke = &jl_fptr_interpret_call; + JL_GC_POP(); + } + JL_UNLOCK(&codegen_lock); // Might GC +} + + +// get a native disassembly for a compiled method extern "C" JL_DLLEXPORT -void jl_add_optimization_passes(LLVMPassManagerRef PM, int opt_level, int lower_intrinsics) { - addOptimizationPasses(unwrap(PM), opt_level, lower_intrinsics); +jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, + int raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo) +{ + // printing via disassembly + jl_code_instance_t *codeinst = jl_generate_fptr(mi, world); + if (codeinst) { + uintptr_t fptr = (uintptr_t)codeinst->invoke; + if (getwrapper) + return jl_dump_fptr_asm(fptr, raw_mc, asm_variant, debuginfo); + uintptr_t specfptr = (uintptr_t)codeinst->specptr.fptr; + if (fptr == (uintptr_t)&jl_fptr_const_return && specfptr == 0) { + // normally we prevent native code from being generated for these functions, + // so create an exception here so we can print pretty lies + JL_LOCK(&codegen_lock); // also disables finalizers, to prevent any unexpected recursion + specfptr = (uintptr_t)codeinst->specptr.fptr; + if (specfptr == 0) { + jl_code_info_t *src = jl_type_infer(mi, world, 0); + JL_GC_PUSH1(&src); + jl_method_t *def = mi->def.method; + if (jl_is_method(def)) { + if (!src) { + // TODO: jl_code_for_staged can throw + src = def->generator ? jl_code_for_staged(mi) : (jl_code_info_t*)def->source; + } + if (src && (jl_value_t*)src != jl_nothing) + src = jl_uncompress_ast(mi->def.method, codeinst, (jl_array_t*)src); + } + fptr = (uintptr_t)codeinst->invoke; + specfptr = (uintptr_t)codeinst->specptr.fptr; + if (src && jl_is_code_info(src)) { + if (fptr == (uintptr_t)&jl_fptr_const_return && specfptr == 0) { + fptr = (uintptr_t)_jl_compile_codeinst(codeinst, src, world); + specfptr = (uintptr_t)codeinst->specptr.fptr; + } + } + JL_GC_POP(); + } + JL_UNLOCK(&codegen_lock); + } + if (specfptr) + return jl_dump_fptr_asm(specfptr, raw_mc, asm_variant, debuginfo); + } + + // whatever, that didn't work - use the assembler output instead + if (raw_mc) // eh, give up, this flag doesn't really work anyways normally + return (jl_value_t*)jl_pchar_to_array("", 0); + return jl_dump_llvm_asm(jl_get_llvmf_defn(mi, world, getwrapper, true, jl_default_cgparams), asm_variant, debuginfo); } #if defined(_OS_LINUX_) || defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_) @@ -301,8 +380,7 @@ void JuliaOJIT::DebugObjectRegistrar::registerObject(RTDyldObjHandleT H, const O const ObjT* Object = &Obj; JIT.NotifyFinalizer(H, *Object, *LO); - ORCNotifyObjectEmitted(JuliaListener.get(), *Object, - *LO, JIT.MemMgr.get()); + ORCNotifyObjectEmitted(JuliaListener.get(), *Object, *LO, JIT.MemMgr.get()); // record all of the exported symbols defined in this object // in the primary hash table for the enclosing JIT @@ -427,6 +505,12 @@ void *JuliaOJIT::getPointerToGlobalIfAvailable(const GlobalValue *GV) void JuliaOJIT::addModule(std::unique_ptr M) { + std::vector NewExports; + for (auto &F : M->functions()) { + if (!F.isDeclaration() && F.getLinkage() == GlobalValue::ExternalLinkage) { + NewExports.push_back(strdup(F.getName().str().c_str())); + } + } #ifndef JL_NDEBUG // validate the relocations for M for (Module::global_object_iterator I = M->global_objects().begin(), E = M->global_objects().end(); I != E; ) { @@ -457,6 +541,11 @@ void JuliaOJIT::addModule(std::unique_ptr M) // Check for errors to prevent LLVM from crashing the program. if (Err) report_fatal_error(std::move(Err)); + // record a stable name for this fptr address + for (auto Name : NewExports) { + void *addr = LocalSymbolTable[getMangledName(Name)]; + ReverseLocalSymbolTable[addr] = Name; + } } void JuliaOJIT::removeModule(ModuleHandleT H) @@ -472,8 +561,11 @@ JL_JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbols Addr = getPointerToGlobalIfAvailable(Name); } // Step 2: Search all previously emitted symbols - if (Addr == nullptr) - Addr = LocalSymbolTable[Name]; + if (Addr == nullptr) { + auto it = LocalSymbolTable.find(Name); + if (it != LocalSymbolTable.end()) + Addr = it->second; + } return JL_JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } @@ -514,11 +606,34 @@ uint64_t JuliaOJIT::getFunctionAddress(const std::string &Name) return addr ? addr.get() : 0; } -Function *JuliaOJIT::FindFunctionNamed(const std::string &Name) +StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *codeinst) { - return shadow_output->getFunction(Name); + auto &fname = ReverseLocalSymbolTable[(void*)(uintptr_t)Addr]; + if (fname.empty()) { + std::stringstream stream_fname; + // try to pick an appropriate name that describes it + if (Addr == (uintptr_t)codeinst->invoke) { + stream_fname << "jsysw_"; + } + else if (codeinst->invoke == &jl_fptr_args) { + stream_fname << "jsys1_"; + } + else if (codeinst->invoke == &jl_fptr_sparam) { + stream_fname << "jsys3_"; + } + else { + stream_fname << "jlsys_"; + } + const char* unadorned_name = jl_symbol_name(codeinst->def->def.method->name); + stream_fname << unadorned_name << "_" << globalUnique++; + std::string string_fname = stream_fname.str(); + fname = strdup(string_fname.c_str()); + LocalSymbolTable[getMangledName(string_fname)] = (void*)(uintptr_t)Addr; + } + return fname; } + void JuliaOJIT::RegisterJITEventListener(JITEventListener *L) { if (!L) @@ -544,7 +659,7 @@ const Triple& JuliaOJIT::getTargetTriple() const return TM.getTargetTriple(); } -std::string JuliaOJIT::getMangledName(const std::string &Name) +std::string JuliaOJIT::getMangledName(StringRef Name) { SmallString<128> FullName; Mangler::getNameWithPrefix(FullName, Name, DL); @@ -558,53 +673,17 @@ std::string JuliaOJIT::getMangledName(const GlobalValue *GV) JuliaOJIT *jl_ExecutionEngine; -// MSVC's link.exe requires each function declaration to have a Comdat section -// So rather than litter the code with conditionals, -// all global values that get emitted call this function -// and it decides whether the definition needs a Comdat section and adds the appropriate declaration -// TODO: consider moving this into jl_add_to_shadow or jl_dump_shadow? the JIT doesn't care, so most calls are now no-ops -template // for GlobalObject's -static T *addComdat(T *G) -{ -#if defined(_OS_WINDOWS_) - if (imaging_mode && !G->isDeclaration()) { - // Add comdat information to make MSVC link.exe happy - // it's valid to emit this for ld.exe too, - // but makes it very slow to link for no benefit - if (G->getParent() == shadow_output) { -#if defined(_COMPILER_MICROSOFT_) - Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); - // ELF only supports Comdat::Any - jl_Comdat->setSelectionKind(Comdat::NoDuplicates); - G->setComdat(jl_Comdat); -#endif -#if defined(_CPU_X86_64_) - // Add unwind exception personalities to functions to handle async exceptions - assert(!juliapersonality_func || juliapersonality_func->getParent() == shadow_output); - if (Function *F = dyn_cast(G)) - F->setPersonalityFn(juliapersonality_func); -#endif - } - // add __declspec(dllexport) to everything marked for export - if (G->getLinkage() == GlobalValue::ExternalLinkage) - G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); - else - G->setDLLStorageClass(GlobalValue::DefaultStorageClass); - } -#endif - return G; -} - // destructively move the contents of src into dest // this assumes that the targets of the two modules are the same // including the DataLayout and ModuleFlags (for example) // and that there is no module-level assembly -static void jl_merge_module(Module *dest, std::unique_ptr src) +// Comdat is also removed, since the JIT doesn't need it +void jl_merge_module(Module *dest, std::unique_ptr src) { assert(dest != src.get()); for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) { GlobalVariable *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + GlobalVariable *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; // Replace a declaration with the definition: if (dG) { @@ -614,6 +693,7 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) continue; } else { + assert(dG->isDeclaration() || dG->getInitializer() == sG->getInitializer()); dG->replaceAllUsesWith(sG); dG->eraseFromParent(); } @@ -621,13 +701,13 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) // Reparent the global variable: sG->removeFromParent(); dest->getGlobalList().push_back(sG); - // Comdat is owned by the Module, recreate it in the new parent: - addComdat(sG); + // Comdat is owned by the Module + sG->setComdat(nullptr); } for (Module::iterator I = src->begin(), E = src->end(); I != E;) { Function *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + Function *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; // Replace a declaration with the definition: if (dG) { @@ -637,6 +717,7 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) continue; } else { + assert(dG->isDeclaration()); dG->replaceAllUsesWith(sG); dG->eraseFromParent(); } @@ -644,13 +725,13 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) // Reparent the global variable: sG->removeFromParent(); dest->getFunctionList().push_back(sG); - // Comdat is owned by the Module, recreate it in the new parent: - addComdat(sG); + // Comdat is owned by the Module + sG->setComdat(nullptr); } for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) { GlobalAlias *sG = &*I; - GlobalValue *dG = dest->getNamedValue(sG->getName()); + GlobalAlias *dG = cast_or_null(dest->getNamedValue(sG->getName())); ++I; if (dG) { if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two @@ -678,144 +759,48 @@ static void jl_merge_module(Module *dest, std::unique_ptr src) } } -// to finalize a function, look up its name in the `module_for_fname` map of -// unfinalized functions and merge it, plus any other modules it depends upon, -// into `collector` then add `collector` to the execution engine -static StringMap module_for_fname; -static void jl_merge_recursive(Module *m, Module *collector); +// this is a unique_ptr, but we don't want +// C++ to attempt to run out finalizer on exit +// since it also owned by jl_LLVMContext +static Module *ready_to_emit; -static void jl_add_to_ee(std::unique_ptr m) +static void jl_add_to_ee() { + JL_TIMING(LLVM_EMIT); + std::unique_ptr m(ready_to_emit); + ready_to_emit = NULL; + if (m) { #if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) - // Add special values used by debuginfo to build the UnwindData table registration for Win64 - ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); + // Add special values used by debuginfo to build the UnwindData table registration for Win64 + Type *T_uint32 = Type::getInt32Ty(m->getContext()); + ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); #endif - assert(jl_ExecutionEngine); - jl_ExecutionEngine->addModule(std::move(m)); -} - -void jl_finalize_function(StringRef F) -{ - std::unique_ptr m(module_for_fname.lookup(F)); - if (m) { - jl_merge_recursive(m.get(), m.get()); - jl_add_to_ee(std::move(m)); - } -} - -static void jl_finalize_function(const std::string &F, Module *collector) -{ - std::unique_ptr m(module_for_fname.lookup(F)); - if (m) { - jl_merge_recursive(m.get(), collector); - jl_merge_module(collector, std::move(m)); - } -} - -static void jl_merge_recursive(Module *m, Module *collector) -{ - // probably not many unresolved declarations, but be sure to iterate over their Names, - // since the declarations may get destroyed by the jl_merge_module call. - // this is also why we copy the Name string, rather than save a StringRef - SmallVector to_finalize; - for (Module::global_object_iterator I = m->global_objects().begin(), E = m->global_objects().end(); I != E; ++I) { - GlobalObject *F = &*I; - if (!F->isDeclaration()) { - module_for_fname.erase(F->getName()); - } - else if (isa(F) && !isIntrinsicFunction(cast(F))) { - to_finalize.push_back(F->getName().str()); - } - else if (isa(F) && module_for_fname.count(F->getName())) { - to_finalize.push_back(F->getName().str()); - } - } - - for (const auto F : to_finalize) { - jl_finalize_function(F, collector); - } -} - -// see if any of the functions needed by F are still WIP -static StringSet<> incomplete_fname; -static bool can_finalize_function(StringRef F, SmallSet &known) -{ - if (incomplete_fname.find(F) != incomplete_fname.end()) - return false; - Module *M = module_for_fname.lookup(F); - if (M && known.insert(M).second) { - for (Module::iterator I = M->begin(), E = M->end(); I != E; ++I) { - Function *F = &*I; - if (F->isDeclaration() && !isIntrinsicFunction(F)) { - if (!can_finalize_function(F->getName(), known)) - return false; - } - } + assert(jl_ExecutionEngine); + jl_ExecutionEngine->addModule(std::move(m)); } - return true; -} -bool jl_can_finalize_function(StringRef F) -{ - SmallSet known; - return can_finalize_function(F, known); } -// let the JIT know this function is a WIP -void jl_init_function(Function *F) +// this passes ownership of a module to the JIT after code emission is complete +void jl_finalize_module(std::unique_ptr m) { - // set any attributes that *must* be set on all functions -#if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - // tell Win32 to realign the stack to the next 16-byte boundary - // upon entry to any function. This achieves compatibility - // with both MinGW-GCC (which assumes an 16-byte-aligned stack) and - // i686 Windows (which uses a 4-byte-aligned stack) - AttrBuilder attr; - attr.addStackAlignmentAttr(16); - F->addAttributes(AttributeList::FunctionIndex, attr); -#endif -#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) - F->setHasUWTable(); // force NeedsWinEH -#endif -#ifdef JL_DISABLE_FPO -#if LLVM_VERSION_MAJOR >= 8 - F->addFnAttr("frame-pointer", "all"); -#else - F->addFnAttr("no-frame-pointer-elim", "true"); -#endif -#endif - // record the WIP name - incomplete_fname.insert(F->getName()); + if (ready_to_emit) + jl_merge_module(ready_to_emit, std::move(m)); + else + ready_to_emit = m.release(); } -// this takes ownership of a module after code emission is complete -// and will add it to the execution engine when required (by jl_finalize_function) -void jl_finalize_module(Module *m, bool shadow) +static uint64_t getAddressForFunction(StringRef fname) { - // record the function names that are part of this Module - // so it can be added to the JIT when needed - for (Module::global_object_iterator I = m->global_objects().begin(), E = m->global_objects().end(); I != E; ++I) { - GlobalObject *F = &*I; - if (!F->isDeclaration()) { - if (isa(F)) { - bool known = incomplete_fname.erase(F->getName()); - (void)known; // TODO: assert(known); // llvmcall gets this wrong - } - module_for_fname[F->getName()] = m; - } - } - // in the newer JITs, the shadow module is separate from the execution module - if (shadow) - jl_add_to_shadow(m); + return jl_ExecutionEngine->getFunctionAddress(fname); } // helper function for adding a DLLImport (dlsym) address to the execution engine -// (for values created locally or in the sysimage, jl_emit_and_add_to_shadow is generally preferable) void add_named_global(GlobalObject *gv, void *addr, bool dllimport) { #ifdef _OS_WINDOWS_ @@ -830,320 +815,3 @@ void add_named_global(GlobalObject *gv, void *addr, bool dllimport) jl_ExecutionEngine->addGlobalMapping(gv, addr); } - -static std::vector jl_sysimg_gvars; -static std::vector jl_sysimg_fvars; -static std::map jl_value_to_llvm; - -// global variables to pointers are pretty common, -// so this method is available as a convenience for emitting them. -// for other types, the formula for implementation is straightforward: -// (see stringConstPtr, for an alternative example to the code below) -// -// if in imaging_mode, emit a GlobalVariable with the same name and an initializer to the shadow_module -// making it valid for emission and reloading in the sysimage -// -// then add a global mapping to the current value (usually from calloc'd space) -// to the execution engine to make it valid for the current session (with the current value) -void** jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit) -{ - PointerType *T = cast(gv->getType()->getElementType()); // pointer is the only supported type here - - GlobalVariable *shadowvar = NULL; - if (imaging_mode) - shadowvar = global_proto(gv, shadow_output); - - if (shadowvar) { - shadowvar->setInitializer(ConstantPointerNull::get(T)); - shadowvar->setLinkage(GlobalVariable::InternalLinkage); - addComdat(shadowvar); - if (imaging_mode && gvarinit) { - // make the pointer valid for future sessions - jl_sysimg_gvars.push_back(shadowvar); - jl_value_llvm gv_struct; - gv_struct.gv = global_proto(gv); - gv_struct.index = jl_sysimg_gvars.size(); - jl_value_to_llvm[gvarinit] = gv_struct; - } - } - - // make the pointer valid for this session - void **slot = (void**)calloc(1, sizeof(void*)); - jl_ExecutionEngine->addGlobalMapping(gv, slot); - return slot; -} - -void* jl_get_globalvar(GlobalVariable *gv) -{ - void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(gv); - assert(p); - return p; -} - -// clones the contents of the module `m` to the shadow_output collector -void jl_add_to_shadow(Module *m) -{ -#ifndef KEEP_BODIES - if (!imaging_mode && !jl_options.outputjitbc) - return; -#endif - ValueToValueMapTy VMap; - std::unique_ptr clone(CloneModule(*m, VMap)); - for (Module::iterator I = clone->begin(), E = clone->end(); I != E; ++I) { - Function *F = &*I; - if (!F->isDeclaration()) { - F->setLinkage(Function::InternalLinkage); - addComdat(F); - } - } - jl_merge_module(shadow_output, std::move(clone)); -} - -static void emit_offset_table(Module *mod, const std::vector &vars, StringRef name) -{ - // Emit a global variable with all the variable addresses. - // The cloning pass will convert them into offsets. - assert(!vars.empty()); - size_t nvars = vars.size(); - std::vector addrs(nvars); - for (size_t i = 0; i < nvars; i++) - addrs[i] = ConstantExpr::getBitCast(vars[i], T_psize); - ArrayType *vars_type = ArrayType::get(T_psize, nvars); - new GlobalVariable(*mod, vars_type, true, - GlobalVariable::ExternalLinkage, - ConstantArray::get(vars_type, addrs), - name); -} - -static void emit_result(std::vector &Archive, SmallVectorImpl &OS, - StringRef Name, std::vector &outputs) -{ - outputs.push_back({ OS.data(), OS.size() }); - Archive.push_back(NewArchiveMember(MemoryBufferRef(outputs.back(), Name))); - OS.clear(); -} - -static object::Archive::Kind getDefaultForHost(Triple &triple) { - if (triple.isOSDarwin()) - return object::Archive::K_DARWIN; - return object::Archive::K_GNU; -} - -typedef Error ArchiveWriterError; -static void reportWriterError(const ErrorInfoBase &E) { - std::string err = E.message(); - jl_safe_printf("ERROR: failed to emit output file %s\n", err.c_str()); -} - -// takes the running content that has collected in the shadow module and dump it to disk -// this builds the object file portion of the sysimage files for fast startup -extern "C" -void jl_dump_native(const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len) -{ - JL_TIMING(NATIVE_DUMP); - // We don't want to use MCJIT's target machine because - // it uses the large code model and we may potentially - // want less optimizations there. - Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); - // make sure to emit the native object format, even if FORCE_ELF was set in codegen -#if defined(_OS_WINDOWS_) - TheTriple.setObjectFormat(Triple::COFF); -#elif defined(_OS_DARWIN_) - TheTriple.setObjectFormat(Triple::MachO); - TheTriple.setOS(llvm::Triple::MacOSX); -#endif - std::unique_ptr - TM(jl_TargetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - jl_TargetMachine->getTargetCPU(), - jl_TargetMachine->getTargetFeatureString(), - jl_TargetMachine->Options, -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - Reloc::PIC_, -#else - Optional(), -#endif -#if defined(_CPU_PPC_) || defined(_CPU_PPC64_) - // On PPC the small model is limited to 16bit offsets - CodeModel::Medium, -#else - // Use small model so that we can use signed 32bits offset in the function and GV tables - CodeModel::Small, -#endif - CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? - )); - - legacy::PassManager PM; - addTargetPasses(&PM, TM.get()); - - // set up optimization passes - SmallVector bc_Buffer; - SmallVector obj_Buffer; - SmallVector unopt_bc_Buffer; - raw_svector_ostream bc_OS(bc_Buffer); - raw_svector_ostream obj_OS(obj_Buffer); - raw_svector_ostream unopt_bc_OS(unopt_bc_Buffer); - std::vector bc_Archive; - std::vector obj_Archive; - std::vector unopt_bc_Archive; - std::vector outputs; - - if (unopt_bc_fname) - PM.add(createBitcodeWriterPass(unopt_bc_OS)); - if (bc_fname || obj_fname) - addOptimizationPasses(&PM, jl_options.opt_level, true, true); - if (bc_fname) - PM.add(createBitcodeWriterPass(bc_OS)); - if (obj_fname) - if (TM->addPassesToEmitFile(PM, obj_OS, nullptr, CGFT_ObjectFile, false)) - jl_safe_printf("ERROR: target does not support generation of object files\n"); - - // Reset the target triple to make sure it matches the new target machine - shadow_output->setTargetTriple(TM->getTargetTriple().str()); - DataLayout DL = TM->createDataLayout(); - DL.reset(DL.getStringRepresentation() + "-ni:10:11:12:13"); - shadow_output->setDataLayout(DL); - - // add metadata information - if (imaging_mode) { - emit_offset_table(shadow_output, jl_sysimg_gvars, "jl_sysimg_gvars"); - emit_offset_table(shadow_output, jl_sysimg_fvars, "jl_sysimg_fvars"); - - // reflect the address of the jl_RTLD_DEFAULT_handle variable - // back to the caller, so that we can check for consistency issues - GlobalValue *jlRTLD_DEFAULT_var = shadow_output->getNamedValue("jl_RTLD_DEFAULT_handle"); - addComdat(new GlobalVariable(*shadow_output, - jlRTLD_DEFAULT_var->getType(), - true, - GlobalVariable::ExternalLinkage, - jlRTLD_DEFAULT_var, - "jl_RTLD_DEFAULT_handle_pointer")); - } - - // do the actual work - auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name) { - PM.run(M); - if (unopt_bc_fname) - emit_result(unopt_bc_Archive, unopt_bc_Buffer, unopt_bc_Name, outputs); - if (bc_fname) - emit_result(bc_Archive, bc_Buffer, bc_Name, outputs); - if (obj_fname) - emit_result(obj_Archive, obj_Buffer, obj_Name, outputs); - }; - - add_output(*shadow_output, "unopt.bc", "text.bc", "text.o"); - // save some memory, by deleting all of the function bodies - for (auto &F : shadow_output->functions()) { - if (!F.isDeclaration()) - F.deleteBody(); - } - - LLVMContext &Context = shadow_output->getContext(); - std::unique_ptr sysimage(new Module("sysimage", Context)); - sysimage->setTargetTriple(shadow_output->getTargetTriple()); - sysimage->setDataLayout(shadow_output->getDataLayout()); - - addComdat(new GlobalVariable(*sysimage, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, globalUnique + 1), - "jl_globalUnique")); - - if (sysimg_data) { - Constant *data = ConstantDataArray::get(Context, - ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); - addComdat(new GlobalVariable(*sysimage, data->getType(), false, - GlobalVariable::ExternalLinkage, - data, "jl_system_image_data"))->setAlignment(Align(64)); - Constant *len = ConstantInt::get(T_size, sysimg_len); - addComdat(new GlobalVariable(*sysimage, len->getType(), true, - GlobalVariable::ExternalLinkage, - len, "jl_system_image_size")); - } - add_output(*sysimage, "data.bc", "data.bc", "data.o"); - - object::Archive::Kind Kind = getDefaultForHost(TheTriple); - if (unopt_bc_fname) - handleAllErrors(writeArchive(unopt_bc_fname, unopt_bc_Archive, true, - Kind, true, false), reportWriterError); - if (bc_fname) - handleAllErrors(writeArchive(bc_fname, bc_Archive, true, - Kind, true, false), reportWriterError); - if (obj_fname) - handleAllErrors(writeArchive(obj_fname, obj_Archive, true, - Kind, true, false), reportWriterError); - - imaging_mode = false; -} - -extern "C" int32_t jl_assign_functionID(const char *fname) -{ - // give the function an index in the constant lookup table - assert(imaging_mode); - if (fname == NULL) - return 0; - jl_sysimg_fvars.push_back(shadow_output->getNamedValue(fname)); - return jl_sysimg_fvars.size(); -} - -extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) -{ - // map a jl_value_t memory location to a GlobalVariable - std::map::iterator it; - it = jl_value_to_llvm.find(p); - if (it == jl_value_to_llvm.end()) - return 0; - return it->second.index; -} - -GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M) -{ - // emit a GlobalVariable for a jl_value_t named "cname" - std::map::iterator it; - // first see if there already is a GlobalVariable for this address - it = jl_value_to_llvm.find(addr); - if (it != jl_value_to_llvm.end()) - return prepare_global_in(M, (llvm::GlobalVariable*)it->second.gv); - - std::stringstream gvname; - gvname << cname << globalUnique++; - // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(*M, T_pjlvalue, - false, GlobalVariable::ExternalLinkage, - NULL, gvname.str()); - *jl_emit_and_add_to_shadow(gv, addr) = addr; - return gv; -} - -// An LLVM module pass that just runs all julia passes in order. Useful for -// debugging -extern "C" void jl_init_codegen(void); -template -class JuliaPipeline : public Pass { -public: - static char ID; - // A bit of a hack, but works - struct TPMAdapter : public PassManagerBase { - PMTopLevelManager *TPM; - TPMAdapter(PMTopLevelManager *TPM) : TPM(TPM) {} - void add(Pass *P) { TPM->schedulePass(P); } - }; - void preparePassManager(PMStack &Stack) override { - (void)jl_init_llvm(); - PMTopLevelManager *TPM = Stack.top()->getTopLevelManager(); - TPMAdapter Adapter(TPM); - addTargetPasses(&Adapter, jl_TargetMachine); - addOptimizationPasses(&Adapter, OptLevel); - } - JuliaPipeline() : Pass(PT_PassManager, ID) {} - Pass *createPrinterPass(raw_ostream &O, const std::string &Banner) const override { - return createPrintModulePass(O, Banner); - } -}; -template<> char JuliaPipeline<0>::ID = 0; -template<> char JuliaPipeline<2>::ID = 0; -template<> char JuliaPipeline<3>::ID = 0; -static RegisterPass> X("juliaO0", "Runs the entire julia pipeline (at -O0)", false, false); -static RegisterPass> Y("julia", "Runs the entire julia pipeline (at -O2)", false, false); -static RegisterPass> Z("juliaO3", "Runs the entire julia pipeline (at -O3)", false, false); diff --git a/src/jitlayers.h b/src/jitlayers.h index 8b049bd751479..fc34e8e0ab789 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -4,6 +4,7 @@ #include #include #include +#include "llvm/IR/LegacyPassManager.h" #include #include "llvm/ExecutionEngine/Orc/CompileUtils.h" @@ -13,9 +14,6 @@ #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/IR/LegacyPassManager.h" -extern legacy::PassManager *jl_globalPM; - #include #include "julia_assert.h" @@ -25,23 +23,64 @@ extern "C" { extern TargetMachine *jl_TargetMachine; extern Module *shadow_output; extern bool imaging_mode; -#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) -extern Function *juliapersonality_func; -#endif - - -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM); void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool lower_intrinsics=true, bool dump_native=false); -void** jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL); -void* jl_get_globalvar(GlobalVariable *gv); -GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M); -void jl_add_to_shadow(Module *m); -void jl_init_function(Function *f); -bool jl_can_finalize_function(StringRef F); -void jl_finalize_function(StringRef F); -void jl_finalize_module(Module *m, bool shadow); +void jl_finalize_module(std::unique_ptr m); +void jl_merge_module(Module *dest, std::unique_ptr src); + +typedef struct _jl_llvm_functions_t { + std::string functionObject; // jlcall llvm Function name + std::string specFunctionObject; // specialized llvm Function name +} jl_llvm_functions_t; + +struct jl_returninfo_t { + llvm::Function *decl; + enum CallingConv { + Boxed = 0, + Register, + SRet, + Union, + Ghosts + } cc; + size_t union_bytes; + size_t union_align; + size_t union_minalign; + unsigned return_roots; +}; + +typedef std::vector> jl_codegen_call_targets_t; +typedef std::tuple, jl_llvm_functions_t> jl_compile_result_t; + +typedef struct { + // outputs + jl_codegen_call_targets_t workqueue; + std::map globals; + std::map ditypes; + std::map llvmtypes; + // inputs + size_t world = 0; + const jl_cgparams_t *params = &jl_default_cgparams; + bool cache = false; +} jl_codegen_params_t; + +jl_compile_result_t jl_emit_code( + jl_method_instance_t *mi, + jl_code_info_t *src, + jl_value_t *jlrettype, + jl_codegen_params_t ¶ms); + +jl_compile_result_t jl_emit_codeinst( + jl_code_instance_t *codeinst, + jl_code_info_t *src, + jl_codegen_params_t ¶ms); + +void jl_compile_workqueue( + std::map &emitted, + jl_codegen_params_t ¶ms); + +Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt, + jl_codegen_params_t ¶ms); // Connect Modules via prototypes, each owned by module `M` static inline GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) @@ -79,7 +118,26 @@ static inline void add_named_global(GlobalObject *gv, T *addr, bool dllimport = add_named_global(gv, (void*)(uintptr_t)addr, dllimport); } -void jl_init_jit(Type *T_pjlvalue_); +static inline Constant *literal_static_pointer_val(const void *p, Type *T) +{ + // this function will emit a static pointer into the generated code + // the generated code will only be valid during the current session, + // and thus, this should typically be avoided in new API's +#if defined(_P64) + return ConstantExpr::getIntToPtr(ConstantInt::get(Type::getInt64Ty(T->getContext()), (uint64_t)p), T); +#else + return ConstantExpr::getIntToPtr(ConstantInt::get(Type::getInt32Ty(T->getContext()), (uint32_t)p), T); +#endif +} + +static const inline char *name_from_method_instance(jl_method_instance_t *li) +{ + return jl_is_method(li->def.method) ? jl_symbol_name(li->def.method->name) : "top-level scope"; +} + + +void jl_init_jit(void); + typedef JITSymbol JL_JITSymbol; // The type that is similar to SymbolInfo on LLVM 4.0 is actually // `JITEvaluatedSymbol`. However, we only use this type when a JITSymbol @@ -137,11 +195,11 @@ class JuliaOJIT { JL_JITSymbol resolveSymbol(const std::string& Name); uint64_t getGlobalValueAddress(const std::string &Name); uint64_t getFunctionAddress(const std::string &Name); - Function *FindFunctionNamed(const std::string &Name); + StringRef getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *codeinst); const DataLayout& getDataLayout() const; const Triple& getTargetTriple() const; private: - std::string getMangledName(const std::string &Name); + std::string getMangledName(StringRef Name); std::string getMangledName(const GlobalValue *GV); TargetMachine &TM; @@ -163,9 +221,9 @@ class JuliaOJIT { SymbolTableT GlobalSymbolTable; SymbolTableT LocalSymbolTable; + DenseMap ReverseLocalSymbolTable; }; extern JuliaOJIT *jl_ExecutionEngine; -JL_DLLEXPORT extern LLVMContext &jl_LLVMContext; Pass *createLowerPTLSPass(bool imaging_mode); Pass *createCombineMulAddPass(); diff --git a/src/jloptions.c b/src/jloptions.c index 858940b8e14a7..c4813638814ff 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -66,7 +66,6 @@ jl_options_t jl_options = { 0, // quiet NULL, // bind-to NULL, // output-bc NULL, // output-unopt-bc - NULL, // output-jit-bc NULL, // output-o NULL, // output-ji NULL, // output-code_coverage @@ -166,7 +165,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_code_coverage, opt_track_allocation, opt_check_bounds, - opt_output_jit_bc, opt_output_unopt_bc, opt_output_bc, opt_depwarn, @@ -222,7 +220,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "check-bounds", required_argument, 0, opt_check_bounds }, { "output-bc", required_argument, 0, opt_output_bc }, { "output-unopt-bc", required_argument, 0, opt_output_unopt_bc }, - { "output-jit-bc", required_argument, 0, opt_output_jit_bc }, { "output-o", required_argument, 0, opt_output_o }, { "output-ji", required_argument, 0, opt_output_ji }, { "output-incremental",required_argument, 0, opt_incremental }, @@ -519,9 +516,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_options.outputbc = optarg; if (!jl_options.image_file_specified) jl_options.image_file = NULL; break; - case opt_output_jit_bc: - jl_options.outputjitbc = optarg; - break; case opt_output_unopt_bc: jl_options.outputunoptbc = optarg; if (!jl_options.image_file_specified) jl_options.image_file = NULL; diff --git a/src/jltypes.c b/src/jltypes.c index 173d44f5908e1..82351788046a4 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1276,8 +1276,6 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value ndt->abstract = dt->abstract; ndt->instance = NULL; ndt->uid = 0; - ndt->struct_decl = NULL; - ndt->ditype = NULL; ndt->size = 0; jl_precompute_memoized_dt(ndt); if (istuple) @@ -1714,7 +1712,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_datatype_type->name->wrapper = (jl_value_t*)jl_datatype_type; jl_datatype_type->super = (jl_datatype_t*)jl_type_type; jl_datatype_type->parameters = jl_emptysvec; - jl_datatype_type->name->names = jl_perm_symsvec(21, + jl_datatype_type->name->names = jl_perm_symsvec(19, "name", "super", "parameters", @@ -1733,26 +1731,22 @@ void jl_init_types(void) JL_GC_DISABLED "isbitstype", "zeroinit", "isinlinealloc", - "has_concrete_subtype", - "llvm::StructType", - "llvm::DIType"); - jl_datatype_type->types = jl_svec(21, + "has_concrete_subtype"); + jl_datatype_type->types = jl_svec(19, jl_typename_type, jl_datatype_type, jl_simplevector_type, jl_simplevector_type, jl_simplevector_type, jl_any_type, // instance + jl_any_type, jl_any_type, jl_any_type, jl_any_type, // properties jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type, jl_any_type); + jl_any_type); jl_datatype_type->instance = NULL; jl_datatype_type->uid = jl_assign_type_uid(); - jl_datatype_type->struct_decl = NULL; - jl_datatype_type->ditype = NULL; jl_datatype_type->abstract = 0; - // NOTE: types should not really be mutable, but the instance and - // struct_decl fields are basically caches, which are mutated. + // NOTE: types are not actually mutable, but we want to ensure they are heap-allocated with stable addresses + // also, the `uid` field gets reset after loading a .ji precompile file jl_datatype_type->mutabl = 1; jl_datatype_type->ninitialized = 3; jl_precompute_memoized_dt(jl_datatype_type); @@ -1771,8 +1765,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, jl_any_type); jl_typename_type->uid = jl_assign_type_uid(); jl_typename_type->instance = NULL; - jl_typename_type->struct_decl = NULL; - jl_typename_type->ditype = NULL; jl_typename_type->abstract = 0; jl_typename_type->mutabl = 1; jl_typename_type->ninitialized = 2; @@ -1793,8 +1785,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type/*uint8*/, jl_any_type/*uint8*/); jl_methtable_type->uid = jl_assign_type_uid(); jl_methtable_type->instance = NULL; - jl_methtable_type->struct_decl = NULL; - jl_methtable_type->ditype = NULL; jl_methtable_type->abstract = 0; jl_methtable_type->mutabl = 1; jl_methtable_type->ninitialized = 4; @@ -1809,8 +1799,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_symbol_type->types = jl_emptysvec; jl_symbol_type->instance = NULL; jl_symbol_type->uid = jl_assign_type_uid(); - jl_symbol_type->struct_decl = NULL; - jl_symbol_type->ditype = NULL; jl_symbol_type->size = 0; jl_symbol_type->abstract = 0; jl_symbol_type->mutabl = 1; @@ -1826,8 +1814,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_simplevector_type->types = jl_emptysvec; jl_simplevector_type->uid = jl_assign_type_uid(); jl_simplevector_type->instance = NULL; - jl_simplevector_type->struct_decl = NULL; - jl_simplevector_type->ditype = NULL; jl_simplevector_type->abstract = 0; jl_simplevector_type->mutabl = 1; jl_simplevector_type->ninitialized = 0; @@ -2194,7 +2180,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_code_instance_type = jl_new_datatype(jl_symbol("CodeInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(11, + jl_perm_symsvec(10, "def", "next", "min_world", @@ -2204,9 +2190,8 @@ void jl_init_types(void) JL_GC_DISABLED "inferred", //"edges", //"absolute_max", - "invoke", "specptr", - "", ""), // function object decls - jl_svec(11, + "isspecsig", "invoke", "specptr"), // function object decls + jl_svec(10, jl_method_instance_type, jl_any_type, jl_ulong_type, @@ -2216,7 +2201,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, //jl_any_type, //jl_bool_type, - jl_any_type, jl_any_type, // fptrs + jl_bool_type, jl_any_type, jl_any_type), // fptrs 0, 1, 1); jl_svecset(jl_code_instance_type->types, 1, jl_code_instance_type); @@ -2319,8 +2304,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_datatype_type->types, 16, jl_bool_type); jl_svecset(jl_datatype_type->types, 17, jl_bool_type); jl_svecset(jl_datatype_type->types, 18, jl_bool_type); - jl_svecset(jl_datatype_type->types, 19, jl_voidpointer_type); - jl_svecset(jl_datatype_type->types, 20, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 1, jl_module_type); jl_svecset(jl_typename_type->types, 6, jl_long_type); jl_svecset(jl_typename_type->types, 3, jl_type_type); @@ -2338,10 +2321,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_methtable_type->types, 10, jl_uint8_type); jl_svecset(jl_method_type->types, 12, jl_method_instance_type); jl_svecset(jl_method_instance_type->types, 5, jl_code_instance_type); - jl_svecset(jl_code_instance_type->types, 7, jl_voidpointer_type); jl_svecset(jl_code_instance_type->types, 8, jl_voidpointer_type); jl_svecset(jl_code_instance_type->types, 9, jl_voidpointer_type); - jl_svecset(jl_code_instance_type->types, 10, jl_voidpointer_type); jl_compute_field_offsets(jl_datatype_type); jl_compute_field_offsets(jl_typename_type); diff --git a/src/julia.h b/src/julia.h index 3f4da989fb45b..c8a27f51ebad4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -230,11 +230,6 @@ JL_EXTENSION typedef union { // 4 interpreter } jl_generic_specptr_t; -typedef struct _jl_llvm_functions_t { - const char *functionObject; // jl_callptr_t llvm Function name - const char *specFunctionObject; // specialized llvm Function name (on sig+rettype) -} jl_llvm_functions_t; - typedef struct _jl_method_instance_t jl_method_instance_t; typedef struct _jl_line_info_node_t { @@ -357,11 +352,9 @@ typedef struct _jl_code_instance_t { //TODO: uint8_t absolute_max; // whether true max world is unknown // compilation state cache + uint8_t isspecsig; // if specptr is a specialized function signature for specTypes->rettype jl_callptr_t invoke; // jlcall entry point jl_generic_specptr_t specptr; // private data for `jlcall entry point` - // names of declarations in the JIT, - // suitable for referencing in LLVM IR - jl_llvm_functions_t functionObjectsDecls; } jl_code_instance_t; // all values are callable as Functions @@ -465,8 +458,6 @@ typedef struct _jl_datatype_t { uint8_t zeroinit; // if one or more fields requires zero-initialization uint8_t isinlinealloc; // if this is allocated inline uint8_t has_concrete_subtype; // If clear, no value will have this datatype - void *struct_decl; //llvm::Type* - void *ditype; // llvm::MDNode* to be used as llvm::DIType(ditype) } jl_datatype_t; typedef struct { @@ -1237,6 +1228,8 @@ STATIC_INLINE jl_value_t *jl_typemap_entry_sig(jl_typemap_t *tmap JL_PROPAGATES_ return (jl_value_t*)((jl_typemap_entry_t*)tmap)->sig; } +JL_DLLEXPORT int jl_isa_compileable_sig(jl_tupletype_t *type, jl_method_t *definition); + // type constructors JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *inmodule); JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name, jl_value_t *lb, jl_value_t *ub); @@ -1605,7 +1598,7 @@ JL_DLLEXPORT const char *jl_pathname_for_handle(void *handle); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); JL_DLLEXPORT void jl_set_sysimg_so(void *handle); -JL_DLLEXPORT ios_t *jl_create_system_image(void); +JL_DLLEXPORT ios_t *jl_create_system_image(void *); JL_DLLEXPORT void jl_save_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); @@ -1939,7 +1932,6 @@ typedef struct { const char *bindto; const char *outputbc; const char *outputunoptbc; - const char *outputjitbc; const char *outputo; const char *outputji; const char *output_code_coverage; @@ -2049,8 +2041,6 @@ typedef struct { // codegen interface ---------------------------------------------------------- typedef struct { - int cached; // can the compiler use/populate the compilation cache? - int track_allocations; // can we track allocations? int code_coverage; // can we measure coverage? int static_alloc; // is the compiler allowed to allocate statically? diff --git a/src/julia_internal.h b/src/julia_internal.h index a5c71774a6895..f478f48f28e65 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -345,6 +345,7 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz); JL_DLLEXPORT void JL_NORETURN jl_throw_out_of_memory_error(void); + JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void); JL_DLLEXPORT int64_t jl_gc_sync_total_bytes(int64_t offset); void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT; @@ -397,12 +398,8 @@ STATIC_INLINE jl_value_t *undefref_check(jl_datatype_t *dt, jl_value_t *v) JL_NO jl_code_info_t *jl_type_infer(jl_method_instance_t *li, size_t world, int force); jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world); -void jl_generate_fptr(jl_code_instance_t *codeinst); -jl_code_instance_t *jl_compile_linfo( - jl_method_instance_t *li JL_PROPAGATES_ROOT, - jl_code_info_t *src JL_MAYBE_UNROOTED, - size_t world, - const jl_cgparams_t *params); +jl_code_instance_t *jl_generate_fptr(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world); +void jl_generate_fptr_for_unspecialized(jl_code_instance_t *unspec); JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_value_t *rettype, size_t min_world, size_t max_world); @@ -533,7 +530,7 @@ void jl_init_types(void) JL_GC_DISABLED; void jl_init_box_caches(void); void jl_init_frontend(void); void jl_init_primitives(void) JL_GC_DISABLED; -void *jl_init_llvm(void); +void jl_init_llvm(void); void jl_init_codegen(void); void jl_init_intrinsic_functions(void); void jl_init_intrinsic_properties(void); @@ -613,10 +610,20 @@ static inline void jl_set_gc_and_wait(void) } #endif -JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, int raw_mc, const char *asm_variant, const char *debuginfo); -void jl_dump_native(const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len); -int32_t jl_get_llvm_gv(jl_value_t *p) JL_NOTSAFEPOINT; -int32_t jl_assign_functionID(const char *fname); +JL_DLLEXPORT jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world, + int raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo); +JL_DLLEXPORT void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params); +JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, int raw_mc, const char* asm_variant, const char *debuginfo); +JL_DLLEXPORT jl_value_t *jl_dump_llvm_asm(void *F, const char* asm_variant, const char *debuginfo); +JL_DLLEXPORT jl_value_t *jl_dump_function_ir(void *f, char strip_ir_metadata, char dump_module, const char *debuginfo); + +void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams); +void jl_dump_native(void *native_code, + const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, + const char *sysimg_data, size_t sysimg_len); +int32_t jl_get_llvm_gv(void *native_code, jl_value_t *p) JL_NOTSAFEPOINT; +void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, + int32_t *func_idx, int32_t *specfunc_idx); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns @@ -625,7 +632,8 @@ JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp); -JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *li, size_t min_world, size_t max_world); +JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); +jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *type, size_t world); JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); diff --git a/src/precompile.c b/src/precompile.c index 5e9bcbd74c7bf..47195148b032f 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -20,28 +20,23 @@ JL_DLLEXPORT int jl_generating_output(void) return jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputji; } -void jl_precompile(int all); +void *jl_precompile(int all); void jl_write_compiler_output(void) { if (!jl_generating_output()) { - if (jl_options.outputjitbc) - jl_dump_native(NULL, jl_options.outputjitbc, NULL, NULL, 0); return; } + void *native_code = NULL; if (!jl_options.incremental) - jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); + native_code = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); if (!jl_module_init_order) { jl_printf(JL_STDERR, "WARNING: --output requested, but no modules defined during run\n"); return; } - if (jl_options.outputjitbc) { - jl_printf(JL_STDERR, "WARNING: --output-jit-bc is meaningless with options for dumping sysimage data\n"); - } - jl_array_t *worklist = jl_module_init_order; JL_GC_PUSH1(&worklist); jl_module_init_order = jl_alloc_vec_any(0); @@ -73,7 +68,7 @@ void jl_write_compiler_output(void) else { ios_t *s = NULL; if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc) - s = jl_create_system_image(); + s = jl_create_system_image(native_code); if (jl_options.outputji) { if (s == NULL) { @@ -90,7 +85,8 @@ void jl_write_compiler_output(void) if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc) { assert(s); - jl_dump_native(jl_options.outputbc, + jl_dump_native(native_code, + jl_options.outputbc, jl_options.outputunoptbc, jl_options.outputo, (const char*)s->buf, (size_t)s->size); @@ -263,6 +259,7 @@ static void _compile_all_deq(jl_array_t *found) if (ucache->invoke != NULL) continue; src = m->source; + assert(src); // TODO: we could now enable storing inferred function pointers in the `unspecialized` cache //src = jl_type_infer(mi, jl_world_counter, 1); //if (ucache->invoke != NULL) @@ -271,8 +268,7 @@ static void _compile_all_deq(jl_array_t *found) // first try to create leaf signatures from the signature declaration and compile those _compile_all_union((jl_value_t*)ml->sig); // then also compile the generic fallback - jl_compile_linfo(mi, (jl_code_info_t*)src, 1, &jl_default_cgparams); - assert(ucache->functionObjectsDecls.functionObject != NULL); + jl_generate_fptr_for_unspecialized(ucache); } JL_GC_POP(); jl_printf(JL_STDERR, "\n"); @@ -283,12 +279,8 @@ static int compile_all_enq__(jl_typemap_entry_t *ml, void *env) jl_array_t *found = (jl_array_t*)env; // method definition -- compile template field jl_method_t *m = ml->func.method; - if (m->source && - (!m->unspecialized || - !m->unspecialized->cache || - (m->unspecialized->cache->functionObjectsDecls.functionObject == NULL && - m->unspecialized->cache->invoke == NULL))) { - // found a lambda that still needs to be compiled + if (m->source) { + // found a method to compile jl_array_ptr_1d_push(found, (jl_value_t*)ml); } return 1; @@ -332,7 +324,7 @@ static int precompile_enq_specialization_(jl_typemap_entry_t *l, void *closure) jl_code_instance_t *codeinst = mi->cache; while (codeinst) { int do_compile = 0; - if (codeinst->functionObjectsDecls.functionObject == NULL && codeinst->invoke != jl_fptr_const_return) { + if (codeinst->invoke != jl_fptr_const_return) { if (codeinst->inferred && codeinst->inferred != jl_nothing && jl_ast_flag_inferred((jl_array_t*)codeinst->inferred) && !jl_ast_flag_inlineable((jl_array_t*)codeinst->inferred)) { @@ -373,28 +365,33 @@ static void precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) void jl_compile_now(jl_method_instance_t *mi); -static void jl_compile_specializations(void) +void *jl_precompile(int all) { + if (all) + jl_compile_all_defs(); // this "found" array will contain function // type signatures that were inferred but haven't been compiled jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); + jl_array_t *m2 = NULL; + jl_method_instance_t *mi = NULL; + JL_GC_PUSH3(&m, &m2, &mi); jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); - // TODO: Ensure stable ordering to make inference problems more reproducible (#29923) + // TODO: Try for a stable ordering to make inference problems more reproducible (#29923) //jl_sort_types((jl_value_t**)jl_array_data(m), jl_array_len(m)); - size_t i, l = jl_array_len(m); - for (i = 0; i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(m, i); - jl_compile_now(mi); + m2 = jl_alloc_vec_any(0); + for (size_t i = 0; i < jl_array_len(m); i++) { + mi = (jl_method_instance_t*)jl_array_ptr_ref(m, i); + size_t min_world = 0; + size_t max_world = ~(size_t)0; + if (!jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->def.method)) + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_world_counter, &min_world, &max_world, 0); + if (mi) + jl_array_ptr_1d_push(m2, (jl_value_t*)mi); } + m = NULL; + void *native_code = jl_create_native(m2, jl_default_cgparams); JL_GC_POP(); -} - -void jl_precompile(int all) -{ - if (all) - jl_compile_all_defs(); - jl_compile_specializations(); + return native_code; } #ifdef __cplusplus diff --git a/src/staticdata.c b/src/staticdata.c index fb9eaa98510cc..a858321fb0c75 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -105,6 +105,8 @@ static arraylist_t reinit_list; // hash of definitions for predefined function pointers static htable_t fptr_to_id; +void *native_functions; + // array of definitions for the predefined function pointers // (reverse of fptr_to_id) // This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C. @@ -494,7 +496,8 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t write_gctaggedfield(s, (uintptr_t)BindingRef << RELOC_TAG_OFFSET); tot += sizeof(void*); size_t binding_reloc_offset = ios_pos(s->s); - record_gvar(s, jl_get_llvm_gv((jl_value_t*)b), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset); + record_gvar(s, jl_get_llvm_gv(native_functions, (jl_value_t*)b), + ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset); write_pointerfield(s, (jl_value_t*)b->name); write_pointerfield(s, b->value); write_pointerfield(s, b->globalref); @@ -614,7 +617,7 @@ static void jl_write_values(jl_serializer_state *s) size_t reloc_offset = ios_pos(s->s); assert(item < layout_table.len && layout_table.items[item] == NULL); layout_table.items[item] = (void*)reloc_offset; - record_gvar(s, jl_get_llvm_gv(v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset); + record_gvar(s, jl_get_llvm_gv(native_functions, v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset); // write data if (jl_is_cpointer(v)) { @@ -784,50 +787,49 @@ static void jl_write_values(jl_serializer_state *s) jl_code_instance_t *newm = (jl_code_instance_t*)&s->s->buf[reloc_offset]; newm->invoke = NULL; + newm->isspecsig = 0; newm->specptr.fptr = NULL; - newm->functionObjectsDecls.functionObject = NULL; - newm->functionObjectsDecls.specFunctionObject = NULL; - uintptr_t fptr_id = JL_API_NULL; - uintptr_t specfptr_id = 0; + int8_t fptr_id = JL_API_NULL; + int8_t builtin_id = 0; if (m->invoke == jl_fptr_const_return) { fptr_id = JL_API_CONST; } else { if (jl_is_method(m->def->def.method)) { - specfptr_id = jl_fptr_id(m->specptr.fptr); - const char *fname = m->functionObjectsDecls.functionObject; - if (specfptr_id) { // found in the table of builtins - assert(specfptr_id >= 2); + builtin_id = jl_fptr_id(m->specptr.fptr); + if (builtin_id) { // found in the table of builtins + assert(builtin_id >= 2); fptr_id = JL_API_BUILTIN; } - else if (fname) { - assert(reloc_offset < INT32_MAX); - if (!strcmp(fname, "jl_fptr_args")) { - fptr_id = JL_API_BOXED; - } - else if (!strcmp(fname, "jl_fptr_sparam")) { - fptr_id = JL_API_WITH_PARAMETERS; - } - else { - int func = jl_assign_functionID(fname); - assert(func > 0); - ios_ensureroom(s->fptr_record, func * sizeof(void*)); - ios_seek(s->fptr_record, (func - 1) * sizeof(void*)); - write_uint32(s->fptr_record, ~reloc_offset); + else { + int32_t invokeptr_id = 0; + int32_t specfptr_id = 0; + jl_get_function_id(native_functions, m, &invokeptr_id, &specfptr_id); // see if we generated code for it + if (invokeptr_id) { + if (invokeptr_id == -1) { + fptr_id = JL_API_BOXED; + } + else if (invokeptr_id == -2) { + fptr_id = JL_API_WITH_PARAMETERS; + } + else { + assert(invokeptr_id > 0); + ios_ensureroom(s->fptr_record, invokeptr_id * sizeof(void*)); + ios_seek(s->fptr_record, (invokeptr_id - 1) * sizeof(void*)); + write_uint32(s->fptr_record, ~reloc_offset); #ifdef _P64 - write_padding(s->fptr_record, 4); + write_padding(s->fptr_record, 4); #endif - } - fname = m->functionObjectsDecls.specFunctionObject; - if (fname) { - int cfunc = jl_assign_functionID(fname); - assert(cfunc > 0); - ios_ensureroom(s->fptr_record, cfunc * sizeof(void*)); - ios_seek(s->fptr_record, (cfunc - 1) * sizeof(void*)); - write_uint32(s->fptr_record, reloc_offset); + } + if (specfptr_id) { + assert(specfptr_id > invokeptr_id && specfptr_id > 0); + ios_ensureroom(s->fptr_record, specfptr_id * sizeof(void*)); + ios_seek(s->fptr_record, (specfptr_id - 1) * sizeof(void*)); + write_uint32(s->fptr_record, reloc_offset); #ifdef _P64 - write_padding(s->fptr_record, 4); + write_padding(s->fptr_record, 4); #endif + } } } } @@ -837,16 +839,14 @@ static void jl_write_values(jl_serializer_state *s) arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, invoke))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + fptr_id)); // relocation target } - if (specfptr_id >= 2) { + if (builtin_id >= 2) { arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, specptr.fptr))); // relocation location - arraylist_push(&s->relocs_list, (void*)(((uintptr_t)BuiltinFunctionRef << RELOC_TAG_OFFSET) + specfptr_id - 2)); // relocation target + arraylist_push(&s->relocs_list, (void*)(((uintptr_t)BuiltinFunctionRef << RELOC_TAG_OFFSET) + builtin_id - 2)); // relocation target } } else if (jl_is_datatype(v)) { jl_datatype_t *dt = (jl_datatype_t*)v; jl_datatype_t *newdt = (jl_datatype_t*)&s->s->buf[reloc_offset]; - newdt->struct_decl = NULL; - newdt->ditype = NULL; if (dt->layout != NULL) { size_t nf = dt->layout->nfields; size_t np = dt->layout->npointers; @@ -881,7 +881,7 @@ static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v) { // since symbols are static, they might not have had a // reference anywhere in the code image other than here - int32_t gv = jl_get_llvm_gv((jl_value_t*)v); + int32_t gv = jl_get_llvm_gv(native_functions, (jl_value_t*)v); if (gv != 0) { uintptr_t item = backref_id(s, v); assert(item >> RELOC_TAG_OFFSET == SymbolRef); @@ -895,7 +895,7 @@ static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v) static void jl_write_gv_int(jl_serializer_state *s, jl_value_t *v) { - int32_t gv = jl_get_llvm_gv((jl_value_t*)v); + int32_t gv = jl_get_llvm_gv(native_functions, (jl_value_t*)v); if (gv != 0) { uintptr_t item = backref_id(s, v); assert(item >> RELOC_TAG_OFFSET == TagRef); @@ -1172,10 +1172,13 @@ static void jl_update_all_fptrs(jl_serializer_state *s) break; } void *fptr = (void*)(base + offset); - if (specfunc) + if (specfunc) { codeinst->specptr.fptr = fptr; - else + codeinst->isspecsig = 1; // TODO: set only if confirmed to be true + } + else { codeinst->invoke = (jl_callptr_t)fptr; + } jl_fptr_to_llvm(fptr, codeinst, specfunc); } } @@ -1285,11 +1288,12 @@ static void jl_prune_type_cache(jl_svec_t *cache) jl_value_t *ti = jl_svecref(cache, i); if (ti == NULL) break; - if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(ti) != 0) + if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(native_functions, ti) != 0) jl_svecset(cache, ins++, ti); else if (jl_is_datatype(ti)) { jl_value_t *singleton = ((jl_datatype_t*)ti)->instance; - if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND || jl_get_llvm_gv(singleton) != 0)) + if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND || + jl_get_llvm_gv(native_functions, singleton) != 0)) jl_svecset(cache, ins++, ti); } } @@ -1431,10 +1435,11 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_gc_enable(en); } -JL_DLLEXPORT ios_t *jl_create_system_image(void) +JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data) { ios_t *f = (ios_t*)malloc_s(sizeof(ios_t)); ios_mem(f, 0); + native_functions = _native_data; jl_save_system_image_to_stream(f); return f; } diff --git a/stdlib/InteractiveUtils/src/codeview.jl b/stdlib/InteractiveUtils/src/codeview.jl index 6f3032e264b88..99874fd5877f5 100644 --- a/stdlib/InteractiveUtils/src/codeview.jl +++ b/stdlib/InteractiveUtils/src/codeview.jl @@ -66,7 +66,7 @@ import Base.CodegenParams # Printing code representations in IR and assembly function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrapper::Bool, strip_ir_metadata::Bool, dump_module::Bool, syntax::Symbol, - optimize::Bool, debuginfo::Symbol=:default, + optimize::Bool, debuginfo::Symbol, params::CodegenParams=CodegenParams()) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") if isa(f, Core.Builtin) @@ -81,13 +81,17 @@ function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrappe meth = Base.func_for_method_checked(meth, ti, env) linfo = ccall(:jl_specializations_get_linfo, Ref{Core.MethodInstance}, (Any, Any, Any, UInt), meth, ti, env, world) # get the code for it - return _dump_function_linfo(linfo, world, native, wrapper, strip_ir_metadata, dump_module, syntax, optimize, debuginfo, params) + if native + str = _dump_function_linfo_native(linfo, world, wrapper, syntax, debuginfo) + else + str = _dump_function_linfo_llvm(linfo, world, wrapper, strip_ir_metadata, dump_module, optimize, debuginfo, params) + end + # TODO: use jl_is_cacheable_sig instead of isdispatchtuple + isdispatchtuple(linfo.specTypes) || (str = "; WARNING: This code may not match what actually runs.\n" * str) + return str end -function _dump_function_linfo(linfo::Core.MethodInstance, world::UInt, native::Bool, wrapper::Bool, - strip_ir_metadata::Bool, dump_module::Bool, syntax::Symbol, - optimize::Bool, debuginfo::Symbol=:default, - params::CodegenParams=CodegenParams()) +function _dump_function_linfo_native(linfo::Core.MethodInstance, world::UInt, wrapper::Bool, syntax::Symbol, debuginfo::Symbol) if syntax !== :att && syntax !== :intel throw(ArgumentError("'syntax' must be either :intel or :att")) end @@ -96,25 +100,27 @@ function _dump_function_linfo(linfo::Core.MethodInstance, world::UInt, native::B elseif debuginfo !== :source && debuginfo !== :none throw(ArgumentError("'debuginfo' must be either :source or :none")) end - if native - llvmf = ccall(:jl_get_llvmf_decl, Ptr{Cvoid}, (Any, UInt, Bool, CodegenParams), linfo, world, wrapper, params) - else - llvmf = ccall(:jl_get_llvmf_defn, Ptr{Cvoid}, (Any, UInt, Bool, Bool, CodegenParams), linfo, world, wrapper, optimize, params) - end - if llvmf == C_NULL - error("could not compile the specified method") - end + str = ccall(:jl_dump_method_asm, Ref{String}, + (Any, UInt, Cint, Bool, Ptr{UInt8}, Ptr{UInt8}), + linfo, world, 0, wrapper, syntax, debuginfo) + return str +end - if native - str = ccall(:jl_dump_function_asm, Ref{String}, - (Ptr{Cvoid}, Cint, Ptr{UInt8}, Ptr{UInt8}), llvmf, 0, syntax, debuginfo) - else - str = ccall(:jl_dump_function_ir, Ref{String}, - (Ptr{Cvoid}, Bool, Bool, Ptr{UInt8}), llvmf, strip_ir_metadata, dump_module, debuginfo) +function _dump_function_linfo_llvm( + linfo::Core.MethodInstance, world::UInt, wrapper::Bool, + strip_ir_metadata::Bool, dump_module::Bool, + optimize::Bool, debuginfo::Symbol, + params::CodegenParams) + if debuginfo === :default + debuginfo = :source + elseif debuginfo !== :source && debuginfo !== :none + throw(ArgumentError("'debuginfo' must be either :source or :none")) end - - # TODO: use jl_is_cacheable_sig instead of isdispatchtuple - isdispatchtuple(linfo.specTypes) || (str = "; WARNING: This code may not match what actually runs.\n" * str) + llvmf = ccall(:jl_get_llvmf_defn, Ptr{Cvoid}, (Any, UInt, Bool, Bool, CodegenParams), linfo, world, wrapper, optimize, params) + llvmf == C_NULL && error("could not compile the specified method") + str = ccall(:jl_dump_function_ir, Ref{String}, + (Ptr{Cvoid}, Bool, Bool, Ptr{UInt8}), + llvmf, strip_ir_metadata, dump_module, debuginfo) return str end diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 820323d986410..b5d2abc39a1b7 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -257,19 +257,6 @@ end @which get_A18434()(1, y=2) @test counter18434 == 2 -@eval function f_invalid(x) - Base.@_noinline_meta - $(Expr(:loopinfo, 1.0f0)) # some expression that throws an error in codegen - x -end - -let _true = Ref(true), g, h - @noinline g() = _true[] ? 0 : h() - @noinline h() = (g(); f_invalid(_true[])) - @test_throws ErrorException @code_native h() # due to a failure to compile f() - @test g() == 0 -end - let _true = Ref(true), f, g, h @noinline f() = ccall((:time, "error_library_doesnt_exist\0"), Cvoid, ()) # should throw error during runtime @noinline g() = _true[] ? 0 : h() @@ -278,6 +265,48 @@ let _true = Ref(true), f, g, h @test_throws ErrorException h() end +# manually generate a broken function, which will break codegen +# and make sure Julia doesn't crash +@eval @noinline f_broken_code() = 0 +let m = which(f_broken_code, ()) + let src = Base.uncompressed_ast(m) + src.code = Any[ + Expr(:meta, :noinline) + Expr(:return, Expr(:invalid)) + ] + m.source = src + end +end +_true = true +# and show that we can still work around it +@noinline g_broken_code() = _true ? 0 : h_broken_code() +@noinline h_broken_code() = (g_broken_code(); f_broken_code()) +let err = tempname(), + old_stderr = stderr, + new_stderr = open(err, "w") + try + redirect_stderr(new_stderr) + println(new_stderr, "start") + flush(new_stderr) + @eval @test occursin("h_broken_code", sprint(code_native, h_broken_code, ())) + Libc.flush_cstdio() + println(new_stderr, "end") + flush(new_stderr) + @eval @test g_broken_code() == 0 + finally + redirect_stderr(old_stderr) + close(new_stderr) + let errstr = read(err, String) + @test startswith(errstr, """start + Internal error: encountered unexpected error during compilation of f_broken_code: + ErrorException(\"unsupported or misplaced expression \"invalid\" in function f_broken_code\") + """) || errstr + @test endswith(errstr, "\nend\n") || errstr + end + rm(err) + end +end + # Issue #33163 A33163(x; y) = x + y B33163(x) = x @@ -335,27 +364,26 @@ ix86 = r"i[356]86" if Sys.ARCH === :x86_64 || occursin(ix86, string(Sys.ARCH)) function linear_foo() - x = 4 - y = 5 + return 5 end rgx = r"%" buf = IOBuffer() - output="" + output = "" #test that the string output is at&t syntax by checking for occurrences of '%'s - code_native(buf,linear_foo,(), syntax = :att) - output=String(take!(buf)) + code_native(buf, linear_foo, (), syntax = :att, debuginfo = :none) + output = String(take!(buf)) @test occursin(rgx, output) #test that the code output is intel syntax by checking it has no occurrences of '%' - code_native(buf,linear_foo,(), syntax = :intel) - output=String(take!(buf)) + code_native(buf, linear_foo, (), syntax = :intel, debuginfo = :none) + output = String(take!(buf)) @test !occursin(rgx, output) - code_native(buf,linear_foo,()) - output=String(take!(buf)) + code_native(buf, linear_foo, ()) + output = String(take!(buf)) @test occursin(rgx, output) end diff --git a/test/llvmcall.jl b/test/llvmcall.jl index adc3542d39a52..4d650b9173d51 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -214,7 +214,7 @@ module LLVMCallFunctionTest function julia_to_llvm(@nospecialize x) isboxed = Ref{UInt8}() - ccall(:julia_type_to_llvm,Ptr{Cvoid},(Any,Ref{UInt8}),x,isboxed) + ccall(:jl_type_to_llvm, Ptr{Cvoid}, (Any, Ref{UInt8}), x, isboxed) end const AnyTy = julia_to_llvm(Any) @@ -241,11 +241,3 @@ module LLVMCallFunctionTest @test global_value_address1() != global_value_address2() end - -# support for calling external functions -let - f() = ccall("time", llvmcall, Cvoid, (Ptr{Cvoid},), C_NULL) - @test_throws ErrorException f() - f() = ccall("extern time", llvmcall, Cvoid, (Ptr{Cvoid},), C_NULL) - f() -end diff --git a/test/llvmcall2.jl b/test/llvmcall2.jl index 9779bed5e620e..cfd20d210bfd7 100644 --- a/test/llvmcall2.jl +++ b/test/llvmcall2.jl @@ -36,3 +36,11 @@ function ceilfloor(x::Float64) return b end @test ceilfloor(7.4) == 8.0 + +# support for calling external functions +begin + f() = ccall("time", llvmcall, Cvoid, (Ptr{Cvoid},), C_NULL) + @test_throws ErrorException f() + g() = ccall("extern time", llvmcall, Cvoid, (Ptr{Cvoid},), C_NULL) + g() +end diff --git a/test/llvmpasses/loopinfo.jl b/test/llvmpasses/loopinfo.jl index e6f3ec9c6b2ef..67dadbf118e41 100644 --- a/test/llvmpasses/loopinfo.jl +++ b/test/llvmpasses/loopinfo.jl @@ -63,10 +63,10 @@ end # CHECK: call void @julia.loopinfo_marker(), {{.*}}, !julia.loopinfo [[LOOPINFO3:![0-9]+]] # LOWER-NOT: call void @julia.loopinfo_marker() # LOWER: br {{.*}}, !llvm.loop [[LOOPID3:![0-9]+]] -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL-NOT: call void @julia_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL-NOT: call void @j_iteration # FINAL: br end end @@ -89,17 +89,17 @@ end # CHECK: call void @julia.loopinfo_marker(), {{.*}}, !julia.loopinfo [[LOOPINFO4:![0-9]+]] # LOWER-NOT: call void @julia.loopinfo_marker() # LOWER: br {{.*}}, !llvm.loop [[LOOPID4:![0-9]+]] -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL: call void @julia_iteration -# FINAL-NOT: call void @julia_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL: call void @j_iteration +# FINAL-NOT: call void @j_iteration end end @@ -110,8 +110,8 @@ function notunroll(J, I) 1 <= j <= I && continue @show (i,j) iteration(i) -# FINAL: call void @julia_iteration -# FINAL-NOT: call void @julia_iteration +# FINAL: call void @j_iteration +# FINAL-NOT: call void @j_iteration end end end diff --git a/test/reflection.jl b/test/reflection.jl index 2250630155b9f..e7d9d6157075c 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -306,25 +306,6 @@ end end @test functionloc(f14346)[2] == @__LINE__() - 4 -# test jl_get_llvm_fptr. We test functions both in and definitely not in the system image -definitely_not_in_sysimg() = nothing -for (f, t) in Any[(definitely_not_in_sysimg, Tuple{}), - (Base.:+, Tuple{Int, Int})] - meth = which(f, t) - tt = Tuple{typeof(f), t.parameters...} - (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), tt, meth.sig)::Core.SimpleVector - @test ti === tt # intersection should be a subtype - world = Core.Compiler.get_world_counter() - linfo = ccall(:jl_specializations_get_linfo, Ref{Core.MethodInstance}, (Any, Any, Any, UInt), meth, tt, env, world) - params = Base.CodegenParams() - llvmf1 = ccall(:jl_get_llvmf_decl, Ptr{Cvoid}, (Any, UInt, Bool, Base.CodegenParams), linfo::Core.MethodInstance, world, true, params) - @test llvmf1 != C_NULL - llvmf2 = ccall(:jl_get_llvmf_decl, Ptr{Cvoid}, (Any, UInt, Bool, Base.CodegenParams), linfo::Core.MethodInstance, world, false, params) - @test llvmf2 != C_NULL - @test ccall(:jl_get_llvm_fptr, Ptr{Cvoid}, (Ptr{Cvoid},), llvmf1) != C_NULL - @test ccall(:jl_get_llvm_fptr, Ptr{Cvoid}, (Ptr{Cvoid},), llvmf2) != C_NULL -end - # issue #15714 # show variable names for slots and suppress spurious type warnings function f15714(array_var15714) @@ -882,7 +863,7 @@ _test_at_locals2(1,1,0.5f0) @noinline f31687_child(i) = f31687_nonexistent(i) f31687_parent() = f31687_child(0) - params = Base.CodegenParams(cached=false) + params = Base.CodegenParams() _dump_function(f31687_parent, Tuple{}, #=native=#false, #=wrapper=#false, #=strip=#false, #=dump_module=#true, #=syntax=#:att, #=optimize=#false, :none, diff --git a/test/staged.jl b/test/staged.jl index 735bf7986e2dd..07ef46d71e7e9 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Random +using InteractiveUtils: code_llvm, code_native @generated function staged_t1(a,b) if a == Int @@ -153,6 +154,7 @@ module TestGeneratedThrow @cfunction(foo, Cvoid, ()) global inited = true end + inited = false end @test TestGeneratedThrow.inited @@ -168,7 +170,7 @@ end @test _g_f_with_inner2(1)(2) == 2 # @generated functions errors -global gf_err_ref = Ref{Int}() +const gf_err_ref = Ref{Int}() gf_err_ref[] = 0 let gf_err, tsk = @async nothing # create a Task for yield to try to run @@ -177,8 +179,9 @@ let gf_err, tsk = @async nothing # create a Task for yield to try to run yield() gf_err_ref[] += 1000 end - @test_throws ErrorException gf_err() - @test_throws ErrorException gf_err() + Expected = ErrorException("task switch not allowed from inside staged nor pure functions") + @test_throws Expected gf_err() + @test_throws Expected gf_err() @test gf_err_ref[] == 4 end @@ -187,14 +190,18 @@ let gf_err2 @generated function gf_err2(::f) where {f} gf_err_ref[] += 1 reflect = f.instance - gf_err_ref[] += 1 - reflect(+, (Int,Int)) + gf_err_ref[] += 10 + reflect(+, (Int, Int)) gf_err_ref[] += 1000 return nothing end - @test_throws ErrorException gf_err2(code_typed) - @test gf_err_ref[] == 4 + Expected = ErrorException("code reflection cannot be used from generated functions") + @test_throws Expected gf_err2(code_typed) + @test_throws Expected gf_err2(code_llvm) + @test_throws Expected gf_err2(code_native) + @test gf_err_ref[] == 66 @test gf_err2(code_lowered) === nothing + @test gf_err_ref[] == 1077 end # issue #15043