From fcbafaa3d62e22c6a6e181696290ac8823a28f44 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 5 Apr 2019 15:23:27 -0400 Subject: [PATCH 01/41] Implement ImmutableArray This rebases #31630 with several fixed and modifications. After #31630, we had originally decided to hold off on said PR in favor of implementing either more efficient layouts for tuples or some sort of variable-sized struct type. However, in the two years since, neither of those have happened (I had a go at improving tuples and made some progress, but there is much still to be done there). In the meantime, all across the package ecosystem, we've seen an increasing creep of pre-allocation and mutating operations, primarily caused by our lack of sufficiently powerful immutable array abstractions and array optimizations. This works fine for the individual packages in question, but it causes a fair bit of trouble when trying to compose these packages with transformation passes such as AD or domain specific optimizations, since many of those passes do not play well with mutation. More generally, we would like to avoid people needing to pierce abstractions for performance reasons. Given these developments, I think it's getting quite important that we start to seriously look at arrays and try to provide performant and well-optimized arrays in the language. More importantly, I think this is somewhat independent from the actual implementation details. To be sure, it would be nice to move more of the array implementation into Julia by making use of one of the abovementioned langugage features, but that is a bit of an orthogonal concern and not absolutely required. This PR provides an `ImmutableArray` type that is identical in functionality and implementation to `Array`, except that it is immutable. Two new intrinsics `Core.arrayfreeze` and `Core.arraythaw` are provided which are semantically copies and turn a mutable array into an immutable array and vice versa. In the original PR, I additionally provided generic functions `freeze` and `thaw` that would simply forward to these intrinsics. However, said generic functions have been omitted from this PR in favor of simply using constructors to go between mutable and immutable arrays at the high level. Generic `freeze`/`thaw` functions can always be added later, once we have a more complete picture of how these functions would work on non-Array datatypes. Some basic compiler support is provided to elide these copies when the compiler can prove that the original object is dead after the copy. For instance, in the following example: ``` function simple() a = Vector{Float64}(undef, 5) for i = 1:5 a[i] = i end ImmutableArray(a) end ``` the compiler will recognize that the array `a` is dead after its use in `ImmutableArray` and the optimized implementation will simply rewrite the type tag in the originally allocated array to now mark it as immutable. It should be pointed out however, that *semantically* there is still no mutation of the original array, this is simply an optimization. At the moment this compiler transform is rather limited, since the analysis requires escape information in order to compute whether or not the copy may be elided. However, more complete escape analysis is being worked on at the moment, so hopefully this analysis should become more powerful in the very near future. I would like to get this cleaned up and merged resonably quickly, and then crowdsource some improvements to the Array APIs more generally. There are still a number of APIs that are quite bound to the notion of mutable `Array`s. StaticArrays and other packages have been inventing conventions for how to generalize those, but we should form a view in Base what those APIs should look like and harmonize them. Having the `ImmutableArray` in Base should help with that. --- base/array.jl | 28 ++++++++++--- base/compiler/optimize.jl | 2 +- base/compiler/ssair/ir.jl | 7 ++++ base/compiler/ssair/passes.jl | 74 +++++++++++++++++++++++++++++++++ base/compiler/tfuncs.jl | 15 +++++++ base/dict.jl | 2 +- base/experimental.jl | 2 + src/Makefile | 2 +- src/builtin_proto.h | 3 ++ src/builtins.c | 61 ++++++++++++++++++++++++++- src/cgutils.cpp | 2 +- src/codegen.cpp | 29 ++++++++++++- src/datatype.c | 7 +++- src/gc.c | 6 ++- src/intrinsics.cpp | 2 +- src/jl_exported_data.inc | 2 + src/jltypes.c | 10 +++++ src/julia.h | 29 ++++++++++--- src/llvm-late-gc-lowering.cpp | 24 +++++++++++ src/llvm-pass-helpers.cpp | 4 +- src/llvm-pass-helpers.h | 1 + src/rtutils.c | 4 +- src/staticdata.c | 11 ++++- test/choosetests.jl | 2 +- test/compiler/immutablearray.jl | 12 ++++++ 25 files changed, 312 insertions(+), 29 deletions(-) create mode 100644 test/compiler/immutablearray.jl diff --git a/base/array.jl b/base/array.jl index 15c354dce6085..f38e2e10c08e2 100644 --- a/base/array.jl +++ b/base/array.jl @@ -147,12 +147,20 @@ function vect(X...) return copyto!(Vector{T}(undef, length(X)), X) end -size(a::Array, d::Integer) = arraysize(a, convert(Int, d)) -size(a::Vector) = (arraysize(a,1),) -size(a::Matrix) = (arraysize(a,1), arraysize(a,2)) -size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(N))::Dims) +const ImmutableArray = Core.ImmutableArray +const IMArray{T,N} = Union{Array{T, N}, ImmutableArray{T,N}} +const IMVector{T} = IMArray{T, 1} +const IMMatrix{T} = IMArray{T, 2} -asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) +ImmutableArray(a::Array) = Core.arrayfreeze(a) +Array(a::ImmutableArray) = Core.arraythaw(a) + +size(a::IMArray, d::Integer) = arraysize(a, convert(Int, d)) +size(a::IMVector) = (arraysize(a,1),) +size(a::IMMatrix) = (arraysize(a,1), arraysize(a,2)) +size(a::IMArray{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(N))::Dims) + +asize_from(a::IMArray, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) allocatedinline(T::Type) = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0)) @@ -223,6 +231,13 @@ function isassigned(a::Array, i::Int...) ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 end +function isassigned(a::ImmutableArray, i::Int...) + @_inline_meta + ii = (_sub2ind(size(a), i...) % UInt) - 1 + @boundscheck ii < length(a) % UInt || return false + ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 +end + ## copy ## """ @@ -895,6 +910,9 @@ function getindex end @eval getindex(A::Array, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) @eval getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) +@eval getindex(A::ImmutableArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) +@eval getindex(A::ImmutableArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) + # Faster contiguous indexing using copyto! for UnitRange and Colon function getindex(A::Array, I::AbstractUnitRange{<:Integer}) @_inline_meta diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 349ed52c01529..62f4e32b90aea 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -307,7 +307,7 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) ir = adce_pass!(ir) #@Base.show ("after_adce", ir) @timeit "type lift" ir = type_lift_pass!(ir) - @timeit "compact 3" ir = compact!(ir) + ir = memory_opt!(ir) #@Base.show ir if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) diff --git a/base/compiler/ssair/ir.jl b/base/compiler/ssair/ir.jl index bc268d33b1a30..7029b3350b6f7 100644 --- a/base/compiler/ssair/ir.jl +++ b/base/compiler/ssair/ir.jl @@ -319,6 +319,13 @@ function setindex!(x::IRCode, @nospecialize(repl), s::SSAValue) return x end +function ssadominates(ir::IRCode, domtree::DomTree, ssa1::Int, ssa2::Int) + bb1 = block_for_inst(ir.cfg, ssa1) + bb2 = block_for_inst(ir.cfg, ssa2) + bb1 == bb2 && return ssa1 < ssa2 + return dominates(domtree, bb1, bb2) +end + # SSA values that need renaming struct OldSSAValue id::Int diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 838ab5bd21755..54d4d46b2c5ab 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1255,3 +1255,77 @@ function cfg_simplify!(ir::IRCode) compact.active_result_bb = length(bb_starts) return finish(compact) end + +function is_allocation(stmt) + isexpr(stmt, :foreigncall) || return false + s = stmt.args[1] + isa(s, QuoteNode) && (s = s.value) + return s === :jl_alloc_array_1d +end + +function memory_opt!(ir::IRCode) + compact = IncrementalCompact(ir, false) + uses = IdDict{Int, Vector{Int}}() + relevant = IdSet{Int}() + revisit = Int[] + function mark_val(val) + isa(val, SSAValue) || return + val.id in relevant && pop!(relevant, val.id) + end + for ((_, idx), stmt) in compact + if isa(stmt, ReturnNode) + isdefined(stmt, :val) || continue + val = stmt.val + if isa(val, SSAValue) && val.id in relevant + (haskey(uses, val.id)) || (uses[val.id] = Int[]) + push!(uses[val.id], idx) + end + continue + end + (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue + if is_allocation(stmt) + push!(relevant, idx) + # TODO: Mark everything else here + continue + end + # TODO: Replace this by interprocedural escape analysis + if is_known_call(stmt, arrayset, compact) + # The value being set escapes, everything else doesn't + mark_val(stmt.args[4]) + arr = stmt.args[3] + if isa(arr, SSAValue) && arr.id in relevant + (haskey(uses, arr.id)) || (uses[arr.id] = Int[]) + push!(uses[arr.id], idx) + end + elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue) + push!(revisit, idx) + else + # For now we assume everything escapes + # TODO: We could handle PhiNodes specially and improve this + for ur in userefs(stmt) + mark_val(ur[]) + end + end + end + ir = finish(compact) + isempty(revisit) && return ir + domtree = construct_domtree(ir.cfg.blocks) + for idx in revisit + # Make sure that the value we reference didn't escape + id = ir.stmts[idx][:inst].args[2].id + (id in relevant) || continue + + # We're ok to steal the memory if we don't dominate any uses + ok = true + for use in uses[id] + if ssadominates(ir, domtree, idx, use) + ok = false + break + end + end + ok || continue + + ir.stmts[idx][:inst].args[1] = Core.mutating_arrayfreeze + end + return ir +end diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 772db09417393..0961d28a4d0ec 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1532,6 +1532,21 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp sv::Union{InferenceState,Nothing}) if f === tuple return tuple_tfunc(argtypes) + elseif f === Core.arrayfreeze || f === Core.arraythaw + if length(argtypes) != 1 + isva && return Any + return Bottom + end + a = widenconst(argtypes[1]) + at = (f === Core.arrayfreeze ? Array : ImmutableArray) + rt = (f === Core.arrayfreeze ? ImmutableArray : Array) + if a <: at + unw = unwrap_unionall(a) + if isa(unw, DataType) + return rewrap_unionall(rt{unw.parameters[1], unw.parameters[2]}, a) + end + end + return rt end if isa(f, IntrinsicFunction) if is_pure_intrinsic_infer(f) && _all(@nospecialize(a) -> isa(a, Const), argtypes) diff --git a/base/dict.jl b/base/dict.jl index 6918677c4f0bb..6d8bea2d33f20 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -372,7 +372,7 @@ end function setindex!(h::Dict{K,V}, v0, key0) where V where K key = convert(K, key0) if !isequal(key, key0) - throw(ArgumentError("$(limitrepr(key0)) is not a valid key for type $K")) + throw(KeyTypeError(K, key0)) end setindex!(h, v0, key) end diff --git a/base/experimental.jl b/base/experimental.jl index 232d2efd11d21..de51c04f56c66 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -11,6 +11,8 @@ module Experimental using Base: Threads, sync_varname using Base.Meta +using Base: ImmutableArray + """ Const(A::Array) diff --git a/src/Makefile b/src/Makefile index c61523a2bacf7..25e8d931d83c9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -261,7 +261,7 @@ $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/builtin_pro $(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h $(BUILDDIR)/jltypes.o $(BUILDDIR)/jltypes.dbg.obj: $(SRCDIR)/builtin_proto.h $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/codegen_shared.h $(BUILDDIR)/julia_version.h -$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/codegen_shared.h $(SRCDIR)/llvm-pass-helpers.h $(BUILDDIR)/llvm-final-gc-lowering.o $(BUILDDIR)/llvm-final-gc-lowering.dbg.obj: $(SRCDIR)/llvm-pass-helpers.h $(BUILDDIR)/llvm-gc-invariant-verifier.o $(BUILDDIR)/llvm-gc-invariant-verifier.dbg.obj: $(SRCDIR)/codegen_shared.h $(BUILDDIR)/llvm-late-gc-lowering.o $(BUILDDIR)/llvm-late-gc-lowering.dbg.obj: $(SRCDIR)/llvm-pass-helpers.h diff --git a/src/builtin_proto.h b/src/builtin_proto.h index 49d3cd7fe87e1..f781658ed55f9 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -51,6 +51,9 @@ DECLARE_BUILTIN(typeassert); DECLARE_BUILTIN(_typebody); DECLARE_BUILTIN(typeof); DECLARE_BUILTIN(_typevar); +DECLARE_BUILTIN(arrayfreeze); +DECLARE_BUILTIN(arraythaw); +DECLARE_BUILTIN(mutating_arrayfreeze); JL_CALLABLE(jl_f_invoke_kwsorter); JL_CALLABLE(jl_f__structtype); diff --git a/src/builtins.c b/src/builtins.c index 32afff52e0b5f..7bccc23cd56df 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1330,7 +1330,9 @@ JL_CALLABLE(jl_f__typevar) JL_CALLABLE(jl_f_arraysize) { JL_NARGS(arraysize, 2, 2); - JL_TYPECHK(arraysize, array, args[0]); + if (!jl_is_arrayish(args[0])) { + jl_type_error("arraysize", (jl_value_t*)jl_array_type, args[0]); + } jl_array_t *a = (jl_array_t*)args[0]; size_t nd = jl_array_ndims(a); JL_TYPECHK(arraysize, long, args[1]); @@ -1369,7 +1371,9 @@ JL_CALLABLE(jl_f_arrayref) { JL_NARGSV(arrayref, 3); JL_TYPECHK(arrayref, bool, args[0]); - JL_TYPECHK(arrayref, array, args[1]); + if (!jl_is_arrayish(args[1])) { + jl_type_error("arrayref", (jl_value_t*)jl_array_type, args[1]); + } jl_array_t *a = (jl_array_t*)args[1]; size_t i = array_nd_index(a, &args[2], nargs - 2, "arrayref"); return jl_arrayref(a, i); @@ -1645,6 +1649,54 @@ JL_CALLABLE(jl_f__equiv_typedef) return equiv_type(args[0], args[1]) ? jl_true : jl_false; } +JL_CALLABLE(jl_f_arrayfreeze) +{ + JL_NARGSV(arrayfreeze, 1); + JL_TYPECHK(arrayfreeze, array, args[0]); + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + JL_GC_PUSH1(&it); + // The idea is to elide this copy if the compiler or runtime can prove that + // doing so is safe to do. + jl_array_t *na = jl_array_copy(a); + jl_set_typeof(na, it); + JL_GC_POP(); + return (jl_value_t*)na; +} + +JL_CALLABLE(jl_f_mutating_arrayfreeze) +{ + // N.B.: These error checks pretend to be arrayfreeze since this is a drop + // in replacement and we don't want to change the visible error type in the + // optimizer + JL_NARGSV(arrayfreeze, 1); + JL_TYPECHK(arrayfreeze, array, args[0]); + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + jl_set_typeof(a, it); + return (jl_value_t*)a; +} + +JL_CALLABLE(jl_f_arraythaw) +{ + JL_NARGSV(arraythaw, 1); + if (((jl_datatype_t*)jl_typeof(args[0]))->name != jl_immutable_array_typename) { + jl_type_error("arraythaw", (jl_value_t*)jl_immutable_array_type, args[0]); + } + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + JL_GC_PUSH1(&it); + // The idea is to elide this copy if the compiler or runtime can prove that + // doing so is safe to do. + jl_array_t *na = jl_array_copy(a); + jl_set_typeof(na, it); + JL_GC_POP(); + return (jl_value_t*)na; +} + // IntrinsicFunctions --------------------------------------------------------- static void (*runtime_fp[num_intrinsics])(void); @@ -1797,6 +1849,10 @@ void jl_init_primitives(void) JL_GC_DISABLED jl_builtin_arrayset = add_builtin_func("arrayset", jl_f_arrayset); jl_builtin_arraysize = add_builtin_func("arraysize", jl_f_arraysize); + jl_builtin_arrayfreeze = add_builtin_func("arrayfreeze", jl_f_arrayfreeze); + jl_builtin_mutating_arrayfreeze = add_builtin_func("mutating_arrayfreeze", jl_f_mutating_arrayfreeze); + jl_builtin_arraythaw = add_builtin_func("arraythaw", jl_f_arraythaw); + // method table utils jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke); @@ -1868,6 +1924,7 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("AbstractArray", (jl_value_t*)jl_abstractarray_type); add_builtin("DenseArray", (jl_value_t*)jl_densearray_type); add_builtin("Array", (jl_value_t*)jl_array_type); + add_builtin("ImmutableArray", (jl_value_t*)jl_immutable_array_type); add_builtin("Expr", (jl_value_t*)jl_expr_type); add_builtin("LineNumberNode", (jl_value_t*)jl_linenumbernode_type); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 94ac4c071770e..f5314863e94f9 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -500,7 +500,7 @@ static Type *_julia_type_to_llvm(jl_codegen_params_t *ctx, jl_value_t *jt, bool if (isboxed) *isboxed = false; if (jt == (jl_value_t*)jl_bottom_type) return T_void; - if (jl_is_concrete_immutable(jt)) { + if (jl_is_concrete_immutable(jt) && !jl_is_arrayish_type(jt)) { if (jl_datatype_nbits(jt) == 0) return T_void; Type *t = _julia_struct_to_llvm(ctx, jt, isboxed); diff --git a/src/codegen.cpp b/src/codegen.cpp index ba583799e1c97..127e4a5cf8adc 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -860,6 +860,15 @@ static const auto pointer_from_objref_func = new JuliaFunction{ Attributes(C, {Attribute::NonNull}), None); }, }; +static const auto mutating_arrayfreeze_func = new JuliaFunction{ + "julia.mutating_arrayfreeze", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue, T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::NoUnwind, Attribute::NoRecurse}), + Attributes(C, {Attribute::NonNull}), + None); }, +}; static const auto jltuple_func = new JuliaFunction{"jl_f_tuple", get_func_sig, get_func_attrs}; static const std::map builtin_func_map = { @@ -894,6 +903,9 @@ static const std::map builtin_func_map = { { &jl_f_arrayset, new JuliaFunction{"jl_f_arrayset", get_func_sig, get_func_attrs} }, { &jl_f_arraysize, new JuliaFunction{"jl_f_arraysize", get_func_sig, get_func_attrs} }, { &jl_f_apply_type, new JuliaFunction{"jl_f_apply_type", get_func_sig, get_func_attrs} }, + { &jl_f_arrayfreeze, new JuliaFunction{"jl_f_arrayfreeze", get_func_sig, get_func_attrs} }, + { &jl_f_arraythaw, new JuliaFunction{"jl_f_arraythaw", get_func_sig, get_func_attrs} }, + { &jl_f_mutating_arrayfreeze,new JuliaFunction{"jl_f_mutating_arrayfreeze", get_func_sig, get_func_attrs} }, }; static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction{"jl_new_opaque_closure_jlcall", get_func_sig, get_func_attrs}; @@ -969,7 +981,7 @@ static bool deserves_retbox(jl_value_t* t) static bool deserves_sret(jl_value_t *dt, Type *T) { assert(jl_is_datatype(dt)); - return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy(); + return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy() && !jl_is_arrayish_type(dt); } @@ -2887,6 +2899,21 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } + else if (f == jl_builtin_mutating_arrayfreeze && nargs == 1) { + const jl_cgval_t &ary = argv[1]; + jl_value_t *aty_dt = jl_unwrap_unionall(ary.typ); + if (jl_is_array_type(aty_dt)) { + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(aty_dt), jl_tparam1(aty_dt)); + *ret = mark_julia_type(ctx, + ctx.builder.CreateCall(prepare_call(mutating_arrayfreeze_func), + { boxed(ctx, ary), + track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)it)) }), true, it); + return true; + } + return false; + } + else if (f == jl_builtin_arrayset && nargs >= 4) { const jl_cgval_t &ary = argv[2]; jl_cgval_t val = argv[3]; diff --git a/src/datatype.c b/src/datatype.c index 1a3ffa78170ac..f751d744dff1c 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -222,7 +222,8 @@ unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) STATIC_INLINE int jl_is_datatype_make_singleton(jl_datatype_t *d) JL_NOTSAFEPOINT { - return (!d->name->abstract && jl_datatype_size(d) == 0 && d != jl_symbol_type && d->name != jl_array_typename && + return (!d->name->abstract && jl_datatype_size(d) == 0 && d != jl_symbol_type && + d->name != jl_array_typename && d->name != jl_immutable_array_typename && d->isconcretetype && !d->name->mutabl); } @@ -389,7 +390,9 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->layout = &opaque_byte_layout; return; } - else if (st == jl_simplevector_type || st == jl_module_type || st->name == jl_array_typename) { + else if (st == jl_simplevector_type || st == jl_module_type || + st->name == jl_array_typename || + st->name == jl_immutable_array_typename) { static const jl_datatype_layout_t opaque_ptr_layout = {0, 1, -1, sizeof(void*), 0, 0}; st->layout = &opaque_ptr_layout; return; diff --git a/src/gc.c b/src/gc.c index f923b826de544..84b1cf695a4cd 100644 --- a/src/gc.c +++ b/src/gc.c @@ -856,7 +856,8 @@ void jl_gc_force_mark_old(jl_ptls_t ptls, jl_value_t *v) JL_NOTSAFEPOINT size_t l = jl_svec_len(v); dtsz = l * sizeof(void*) + sizeof(jl_svec_t); } - else if (dt->name == jl_array_typename) { + else if (dt->name == jl_array_typename || + dt->name == jl_immutable_array_typename) { jl_array_t *a = (jl_array_t*)v; if (!a->flags.pooled) dtsz = GC_MAX_SZCLASS + 1; @@ -2520,7 +2521,8 @@ mark: { objary = (gc_mark_objarray_t*)sp.data; goto objarray_loaded; } - else if (vt->name == jl_array_typename) { + else if (vt->name == jl_array_typename || + vt->name == jl_immutable_array_typename) { jl_array_t *a = (jl_array_t*)new_obj; jl_array_flags_t flags = a->flags; if (update_meta) { diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index e1d821a34e42d..2c1b8a55754a3 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1073,7 +1073,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar case arraylen: { const jl_cgval_t &x = argv[0]; jl_value_t *typ = jl_unwrap_unionall(x.typ); - if (!jl_is_datatype(typ) || ((jl_datatype_t*)typ)->name != jl_array_typename) + if (!jl_is_arrayish_type(typ)) return emit_runtime_call(ctx, f, argv, nargs); return mark_julia_type(ctx, emit_arraylen(ctx, x), false, jl_long_type); } diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 3cebe459bf643..32dd6cb5079e4 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -15,6 +15,8 @@ XX(jl_array_symbol_type) \ XX(jl_array_type) \ XX(jl_array_typename) \ + XX(jl_immutable_array_type) \ + XX(jl_immutable_array_typename) \ XX(jl_array_uint8_type) \ XX(jl_atomicerror_type) \ XX(jl_base_module) \ diff --git a/src/jltypes.c b/src/jltypes.c index 1ae49c0a32eab..f0efc0e37ce42 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2218,6 +2218,15 @@ void jl_init_types(void) JL_GC_DISABLED jl_nonfunction_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any; jl_type_type_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any; + tv = jl_svec2(tvar("T"), tvar("N")); + jl_immutable_array_type = (jl_unionall_t*) + jl_new_datatype(jl_symbol("ImmutableArray"), core, + (jl_datatype_t*) + jl_apply_type((jl_value_t*)jl_densearray_type, jl_svec_data(tv), 2), + tv, jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 0, 0)->name->wrapper; + jl_immutable_array_typename = ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_immutable_array_type))->name; + jl_compute_field_offsets((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_immutable_array_type)); + jl_expr_type = jl_new_datatype(jl_symbol("Expr"), core, jl_any_type, jl_emptysvec, @@ -2629,6 +2638,7 @@ void jl_init_types(void) JL_GC_DISABLED // override the preferred layout for a couple types jl_lineinfonode_type->name->mayinlinealloc = 0; // FIXME: assumed to be a pointer by codegen + jl_immutable_array_typename->mayinlinealloc = 0; // It seems like we probably usually end up needing the box for kinds (used in an Any context)--but is that true? jl_uniontype_type->name->mayinlinealloc = 0; jl_unionall_type->name->mayinlinealloc = 0; diff --git a/src/julia.h b/src/julia.h index 9afd7301fc5bc..292083225b30e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -657,6 +657,8 @@ extern JL_DLLIMPORT jl_unionall_t *jl_abstractarray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_densearray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_array_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_typename_t *jl_array_typename JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_unionall_t *jl_immutable_array_type JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_typename_t *jl_immutable_array_typename JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_weakref_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abstractstring_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_string_type JL_GLOBALLY_ROOTED; @@ -1200,11 +1202,25 @@ STATIC_INLINE int jl_is_primitivetype(void *v) JL_NOTSAFEPOINT jl_datatype_size(v) > 0); } +STATIC_INLINE int jl_is_array_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + (((jl_datatype_t*)(t))->name == jl_array_typename)); +} + +STATIC_INLINE int jl_is_arrayish_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + (((jl_datatype_t*)(t))->name == jl_array_typename || + ((jl_datatype_t*)(t))->name == jl_immutable_array_typename)); +} + STATIC_INLINE int jl_is_structtype(void *v) JL_NOTSAFEPOINT { return (jl_is_datatype(v) && !((jl_datatype_t*)(v))->name->abstract && - !jl_is_primitivetype(v)); + !jl_is_primitivetype(v) && + !jl_is_arrayish_type(v)); } STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbits() in julia @@ -1222,16 +1238,16 @@ STATIC_INLINE int jl_is_abstracttype(void *v) JL_NOTSAFEPOINT return (jl_is_datatype(v) && ((jl_datatype_t*)(v))->name->abstract); } -STATIC_INLINE int jl_is_array_type(void *t) JL_NOTSAFEPOINT +STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT { - return (jl_is_datatype(t) && - ((jl_datatype_t*)(t))->name == jl_array_typename); + jl_value_t *t = jl_typeof(v); + return jl_is_array_type(t); } -STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT +STATIC_INLINE int jl_is_arrayish(void *v) JL_NOTSAFEPOINT { jl_value_t *t = jl_typeof(v); - return jl_is_array_type(t); + return jl_is_arrayish_type(t); } @@ -1500,6 +1516,7 @@ JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a); JL_DLLEXPORT jl_array_t *jl_alloc_vec_any(size_t n); JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i); // 0-indexed JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; // 0-indexed +JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i); // 0-indexed JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i); // 0-indexed JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i); // 0-indexed diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index d8ad3d62d4cc1..fa73cb54b9b5e 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -359,6 +359,7 @@ struct LateLowerGCFrame: public FunctionPass, private JuliaPassContext { void RefineLiveSet(BitVector &LS, State &S, const std::vector &CalleeRoots); Value *EmitTagPtr(IRBuilder<> &builder, Type *T, Value *V); Value *EmitLoadTag(IRBuilder<> &builder, Value *V); + Value *EmitStoreTag(IRBuilder<> &builder, Value *V, Value *Typ); }; static unsigned getValueAddrSpace(Value *V) { @@ -2186,6 +2187,16 @@ Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Value *V) return load; } +Value *LateLowerGCFrame::EmitStoreTag(IRBuilder<> &builder, Value *V, Value *Typ) +{ + auto addr = EmitTagPtr(builder, T_size, V); + StoreInst *store = builder.CreateAlignedStore(Typ, addr, Align(sizeof(size_t))); + store->setOrdering(AtomicOrdering::Unordered); + store->setMetadata(LLVMContext::MD_tbaa, tbaa_tag); + return store; +} + + // Enable this optimization only on LLVM 4.0+ since this cause LLVM to optimize // constant store loop to produce a `memset_pattern16` with a global variable // that's initialized by `addrspacecast`. Such a global variable is not supported by the backend. @@ -2358,6 +2369,19 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S) { typ->takeName(CI); CI->replaceAllUsesWith(typ); UpdatePtrNumbering(CI, typ, S); + } else if (mutating_arrayfreeze_func && callee == mutating_arrayfreeze_func) { + assert(CI->getNumArgOperands() == 2); + IRBuilder<> builder(CI); + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + auto array = CI->getArgOperand(0); + auto tag = EmitLoadTag(builder, array); + auto mark_bits = builder.CreateAnd(tag, ConstantInt::get(T_size, (uintptr_t)15)); + auto new_typ = builder.CreateAddrSpaceCast(CI->getArgOperand(1), + T_pjlvalue); + auto new_typ_marked = builder.CreateOr(builder.CreatePtrToInt(new_typ, T_size), mark_bits); + EmitStoreTag(builder, array, new_typ_marked); + CI->replaceAllUsesWith(array); + UpdatePtrNumbering(CI, array, S); } else if (write_barrier_func && callee == write_barrier_func) { // The replacement for this requires creating new BasicBlocks // which messes up the loop. Queue all of them to be replaced later. diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 0eed7aec98f0b..eea7b072284ff 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -27,7 +27,8 @@ JuliaPassContext::JuliaPassContext() T_ppjlvalue_der(nullptr), pgcstack_getter(nullptr), gc_flush_func(nullptr), gc_preserve_begin_func(nullptr), gc_preserve_end_func(nullptr), pointer_from_objref_func(nullptr), alloc_obj_func(nullptr), - typeof_func(nullptr), write_barrier_func(nullptr), module(nullptr) + typeof_func(nullptr), mutating_arrayfreeze_func(nullptr), + write_barrier_func(nullptr), module(nullptr) { tbaa_gcframe = tbaa_make_child("jtbaa_gcframe").first; MDNode *tbaa_data; @@ -46,6 +47,7 @@ void JuliaPassContext::initFunctions(Module &M) gc_preserve_end_func = M.getFunction("llvm.julia.gc_preserve_end"); pointer_from_objref_func = M.getFunction("julia.pointer_from_objref"); typeof_func = M.getFunction("julia.typeof"); + mutating_arrayfreeze_func = M.getFunction("julia.mutating_arrayfreeze"); write_barrier_func = M.getFunction("julia.write_barrier"); alloc_obj_func = M.getFunction("julia.gc_alloc_obj"); } diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index f80786d1e7149..9352d01e2fbe9 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -67,6 +67,7 @@ struct JuliaPassContext { llvm::Function *pointer_from_objref_func; llvm::Function *alloc_obj_func; llvm::Function *typeof_func; + llvm::Function *mutating_arrayfreeze_func; llvm::Function *write_barrier_func; // Creates a pass context. Type and function pointers diff --git a/src/rtutils.c b/src/rtutils.c index 67d17c39c67ec..793a5757b10c4 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -988,8 +988,8 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, ")"); } } - else if (jl_array_type && jl_is_array_type(vt)) { - n += jl_printf(out, "Array{"); + else if (jl_array_type && jl_is_arrayish_type(vt)) { + n += jl_printf(out, jl_is_array_type(vt) ? "Array{" : "ImmutableArray{"); n += jl_static_show_x(out, (jl_value_t*)jl_tparam0(vt), depth); n += jl_printf(out, ", ("); size_t i, ndims = jl_array_ndims(v); diff --git a/src/staticdata.c b/src/staticdata.c index 8fa1613b075a8..53e83a0aa2df9 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -30,7 +30,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 150 +#define NUM_TAGS 155 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -50,6 +50,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_slotnumber_type); INSERT_TAG(jl_simplevector_type); INSERT_TAG(jl_array_type); + INSERT_TAG(jl_immutable_array_type); INSERT_TAG(jl_typedslot_type); INSERT_TAG(jl_expr_type); INSERT_TAG(jl_globalref_type); @@ -133,6 +134,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_pointer_typename); INSERT_TAG(jl_llvmpointer_typename); INSERT_TAG(jl_array_typename); + INSERT_TAG(jl_immutable_array_typename); INSERT_TAG(jl_type_typename); INSERT_TAG(jl_namedtuple_typename); INSERT_TAG(jl_vecelement_typename); @@ -200,6 +202,9 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin__expr); INSERT_TAG(jl_builtin_ifelse); INSERT_TAG(jl_builtin__typebody); + INSERT_TAG(jl_builtin_arrayfreeze); + INSERT_TAG(jl_builtin_mutating_arrayfreeze); + INSERT_TAG(jl_builtin_arraythaw); // All optional tags must be placed at the end, so that we // don't accidentally have a `NULL` in the middle @@ -250,7 +255,9 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_opaque_closure_call, - NULL }; + &jl_f_arrayfreeze, &jl_f_mutating_arrayfreeze, &jl_f_arraythaw, + NULL +}; typedef struct { ios_t *s; diff --git a/test/choosetests.jl b/test/choosetests.jl index 21f313fdbbb34..7904df7d405b2 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -100,7 +100,7 @@ function choosetests(choices = []) filtertests!(tests, "subarray") filtertests!(tests, "compiler", ["compiler/inference", "compiler/validation", "compiler/ssair", "compiler/irpasses", "compiler/codegen", - "compiler/inline", "compiler/contextual"]) + "compiler/inline", "compiler/contextual", "compiler/immutablearray"]) filtertests!(tests, "stdlib", STDLIBS) # do ambiguous first to avoid failing if ambiguities are introduced by other tests filtertests!(tests, "ambiguous") diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl new file mode 100644 index 0000000000000..474e5dfc0f657 --- /dev/null +++ b/test/compiler/immutablearray.jl @@ -0,0 +1,12 @@ +using Base.Experimental: ImmutableArray +function simple() + a = Vector{Float64}(undef, 5) + for i = 1:5 + a[i] = i + end + ImmutableArray(a) +end +let + @allocated(simple()) + @test @allocated(simple()) < 100 +end From 233bb0c81239a86705d6ea306f69617250b414f5 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Wed, 29 Sep 2021 18:33:29 -0400 Subject: [PATCH 02/41] Implement maybecopy in BoundsError, start optimization, refactor memory_opt! --- base/abstractarray.jl | 4 + base/array.jl | 53 +++- base/boot.jl | 13 +- base/broadcast.jl | 13 + base/compiler/optimize.jl | 1 + base/compiler/ssair/passes.jl | 162 ++++++++++-- base/pointer.jl | 1 + src/builtin_proto.h | 1 + src/builtins.c | 14 + src/codegen.cpp | 4 +- src/staticdata.c | 5 +- test/compiler/immutablearray.jl | 456 +++++++++++++++++++++++++++++++- 12 files changed, 685 insertions(+), 42 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d5d47fe855bd5..20d49e0bfbcf3 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1073,6 +1073,10 @@ function copy(a::AbstractArray) copymutable(a) end +function copy(a::Core.ImmutableArray) + a +end + function copyto!(B::AbstractVecOrMat{R}, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, A::AbstractVecOrMat{S}, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) where {R,S} if length(ir_dest) != length(ir_src) diff --git a/base/array.jl b/base/array.jl index f38e2e10c08e2..dc22cc4a21cb0 100644 --- a/base/array.jl +++ b/base/array.jl @@ -118,6 +118,35 @@ Union type of [`DenseVector{T}`](@ref) and [`DenseMatrix{T}`](@ref). """ const DenseVecOrMat{T} = Union{DenseVector{T}, DenseMatrix{T}} +""" + ImmutableArray + +Dynamically allocated, immutable array. + +""" +const ImmutableArray = Core.ImmutableArray + +""" + IMArray{T,N} + +Union type of [`Array{T,N}`](@ref) and [`ImmutableArray{T,N}`](@ref) +""" +const IMArray{T,N} = Union{Array{T, N}, ImmutableArray{T,N}} + +""" + IMVector{T} + +One-dimensional [`ImmutableArray`](@ref) or [`Array`](@ref) with elements of type `T`. Alias for `IMArray{T, 1}`. +""" +const IMVector{T} = IMArray{T, 1} + +""" + IMMatrix{T} + +Two-dimensional [`ImmutableArray`](@ref) or [`Array`](@ref) with elements of type `T`. Alias for `IMArray{T,2}`. +""" +const IMMatrix{T} = IMArray{T, 2} + ## Basic functions ## import Core: arraysize, arrayset, arrayref, const_arrayref @@ -147,14 +176,13 @@ function vect(X...) return copyto!(Vector{T}(undef, length(X)), X) end -const ImmutableArray = Core.ImmutableArray -const IMArray{T,N} = Union{Array{T, N}, ImmutableArray{T,N}} -const IMVector{T} = IMArray{T, 1} -const IMMatrix{T} = IMArray{T, 2} - +# Freeze and thaw constructors ImmutableArray(a::Array) = Core.arrayfreeze(a) Array(a::ImmutableArray) = Core.arraythaw(a) +ImmutableArray(a::AbstractArray{T,N}) where {T,N} = ImmutableArray{T,N}(a) + +# Size functions for arrays, both mutable and immutable size(a::IMArray, d::Integer) = arraysize(a, convert(Int, d)) size(a::IMVector) = (arraysize(a,1),) size(a::IMMatrix) = (arraysize(a,1), arraysize(a,2)) @@ -393,6 +421,18 @@ similar(a::Array{T}, m::Int) where {T} = Vector{T}(undef, m) similar(a::Array, T::Type, dims::Dims{N}) where {N} = Array{T,N}(undef, dims) similar(a::Array{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) +ImmutableArray{T}(undef::UndefInitializer, m::Int) where T = ImmutableArray(Array{T}(undef, m)) +ImmutableArray{T}(undef::UndefInitializer, dims::Dims) where T = ImmutableArray(Array{T}(undef, dims)) + +""" + maybecopy(x) + +`maybecopy` provides access to `x` while ensuring it does not escape. +To do so, the optimizer decides whether to create a copy of `x` or not based on the implementation +That is, `maybecopy` will either be a call to [`copy`](@ref) or just a reference to x. +""" +maybecopy = Core.maybecopy + # T[x...] constructs Array{T,1} """ getindex(type[, elements...]) @@ -626,8 +666,8 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) ## Conversions ## -convert(::Type{T}, a::AbstractArray) where {T<:Array} = a isa T ? a : T(a) convert(::Type{Union{}}, a::AbstractArray) = throw(MethodError(convert, (Union{}, a))) +convert(T::Union{Type{<:Array},Type{<:Core.ImmutableArray}}, a::AbstractArray) = a isa T ? a : T(a) promote_rule(a::Type{Array{T,n}}, b::Type{Array{S,n}}) where {T,n,S} = el_same(promote_type(T,S), a, b) @@ -637,6 +677,7 @@ if nameof(@__MODULE__) === :Base # avoid method overwrite # constructors should make copies Array{T,N}(x::AbstractArray{S,N}) where {T,N,S} = copyto_axcheck!(Array{T,N}(undef, size(x)), x) AbstractArray{T,N}(A::AbstractArray{S,N}) where {T,N,S} = copyto_axcheck!(similar(A,T), A) +ImmutableArray{T,N}(Ar::AbstractArray{S,N}) where {T,N,S} = Core.arrayfreeze(copyto_axcheck!(Array{T,N}(undef, size(Ar)), Ar)) end ## copying iterators to containers diff --git a/base/boot.jl b/base/boot.jl index 98b8cf2e9cf40..d59af90d54f13 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -279,8 +279,17 @@ struct BoundsError <: Exception a::Any i::Any BoundsError() = new() - BoundsError(@nospecialize(a)) = (@_noinline_meta; new(a)) - BoundsError(@nospecialize(a), i) = (@_noinline_meta; new(a,i)) + # For now, always copy arrays to avoid escaping them + # Eventually, we want to figure out if the copy is needed to save the performance of copying + # (i.e., if a escapes elsewhere, don't bother to make a copy) + + BoundsError(@nospecialize(a)) = a isa Array ? + (@_noinline_meta; new(Core.maybecopy(a))) : + (@_noinline_meta; new(a)) + + BoundsError(@nospecialize(a), i) = a isa Array ? + (@_noinline_meta; new(Core.maybecopy(a), i)) : + (@_noinline_meta; new(a, i)) end struct DivideError <: Exception end struct OutOfMemoryError <: Exception end diff --git a/base/broadcast.jl b/base/broadcast.jl index b34a73041708b..83067ed46f5e4 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -1385,4 +1385,17 @@ function Base.show(io::IO, op::BroadcastFunction) end Base.show(io::IO, ::MIME"text/plain", op::BroadcastFunction) = show(io, op) +struct IMArrayStyle <: Broadcast.AbstractArrayStyle{Any} end +BroadcastStyle(::Type{<:Core.ImmutableArray}) = IMArrayStyle() + +#similar has to return mutable array +function Base.similar(bc::Broadcasted{IMArrayStyle}, ::Type{ElType}) where ElType + similar(Array{ElType}, axes(bc)) +end + +@inline function copy(bc::Broadcasted{IMArrayStyle}) + ElType = combine_eltypes(bc.f, bc.args) + return Core.ImmutableArray(copyto!(similar(bc, ElType), bc)) +end + end # module diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 62f4e32b90aea..de6294393d3c1 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -307,6 +307,7 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) ir = adce_pass!(ir) #@Base.show ("after_adce", ir) @timeit "type lift" ir = type_lift_pass!(ir) + #@timeit "compact 3" ir = compact!(ir) ir = memory_opt!(ir) #@Base.show ir if JLOptions().debug_level == 2 diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 54d4d46b2c5ab..723776f49fe37 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1256,76 +1256,184 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end -function is_allocation(stmt) +# function is_known_fcall(stmt::Expr, @nospecialize(func)) +# isexpr(stmt, :foreigncall) || return false +# s = stmt.args[1] +# isa(s, QuoteNode) && (s = s.value) +# return s === func +# end + +function is_known_fcall(stmt::Expr, funcs::Vector{Symbol}) isexpr(stmt, :foreigncall) || return false s = stmt.args[1] isa(s, QuoteNode) && (s = s.value) - return s === :jl_alloc_array_1d + # return any(e -> s === e, funcs) + return true in map(e -> s === e, funcs) +end + +function is_allocation(stmt::Expr) + isexpr(stmt, :foreigncall) || return false + s = stmt.args[1] + isa(s, QuoteNode) && (s = s.value) + return (s === :jl_alloc_array_1d + || s === :jl_alloc_array_2d + || s === :jl_alloc_array_3d + || s === :jl_new_array) end function memory_opt!(ir::IRCode) compact = IncrementalCompact(ir, false) uses = IdDict{Int, Vector{Int}}() - relevant = IdSet{Int}() - revisit = Int[] - function mark_val(val) + relevant = IdSet{Int}() # allocations + revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in + maybecopies = Int[] # calls to maybecopy + + function mark_escape(@nospecialize val) isa(val, SSAValue) || return + #println(val.id, " escaped.") val.id in relevant && pop!(relevant, val.id) end + + function mark_use(val, idx) + isa(val, SSAValue) || return + id = val.id + id in relevant || return + (haskey(uses, id)) || (uses[id] = Int[]) + push!(uses[id], idx) + end + for ((_, idx), stmt) in compact + + #println("idx: ", idx, " = ", stmt) + if isa(stmt, ReturnNode) isdefined(stmt, :val) || continue val = stmt.val - if isa(val, SSAValue) && val.id in relevant - (haskey(uses, val.id)) || (uses[val.id] = Int[]) - push!(uses[val.id], idx) - end + mark_use(val, idx) continue + + # check for phinodes that are possibly allocations + elseif isa(stmt, PhiNode) + + # ensure all of the phinode values are defined + defined = true + for i = 1:length(stmt.values) + if !isassigned(stmt.values, i) + defined = false + end + end + + defined || continue + + for val in stmt.values + if isa(val, SSAValue) && val.id in relevant + push!(relevant, idx) + end + end end + (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue + + if is_known_call(stmt, Core.maybecopy, compact) + push!(maybecopies, idx) + continue + end + if is_allocation(stmt) push!(relevant, idx) # TODO: Mark everything else here continue end - # TODO: Replace this by interprocedural escape analysis - if is_known_call(stmt, arrayset, compact) + + if is_known_call(stmt, arrayset, compact) && length(stmt.args) >= 5 # The value being set escapes, everything else doesn't - mark_val(stmt.args[4]) + mark_escape(stmt.args[4]) arr = stmt.args[3] - if isa(arr, SSAValue) && arr.id in relevant - (haskey(uses, arr.id)) || (uses[arr.id] = Int[]) - push!(uses[arr.id], idx) - end + mark_use(arr, idx) + + elseif is_known_call(stmt, arrayref, compact) && length(stmt.args) == 4 + arr = stmt.args[3] + mark_use(arr, idx) + + elseif is_known_call(stmt, setindex!, compact) && length(stmt.args) == 4 + # handle similarly to arrayset + val = stmt.args[3] + mark_escape(val) + + arr = stmt.args[2] + mark_use(arr, idx) + + elseif is_known_call(stmt, (===), compact) && length(stmt.args) == 3 + arr1 = stmt.args[2] + arr2 = stmt.args[3] + + mark_use(arr1, idx) + mark_use(arr2, idx) + + # these foreigncalls have similar structure and don't escape our array, so handle them all at once + elseif is_known_fcall(stmt, [:jl_array_ptr, :jl_array_copy]) && length(stmt.args) == 6 + arr = stmt.args[6] + mark_use(arr, idx) + + elseif is_known_call(stmt, arraysize, compact) && isa(stmt.args[2], SSAValue) + arr = stmt.args[2] + mark_use(arr, idx) + elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue) + # mark these for potential replacement with mutating_arrayfreeze push!(revisit, idx) + else - # For now we assume everything escapes - # TODO: We could handle PhiNodes specially and improve this + # Assume everything else escapes for ur in userefs(stmt) - mark_val(ur[]) + mark_escape(ur[]) end end end + ir = finish(compact) - isempty(revisit) && return ir + isempty(revisit) && isempty(maybecopies) && return ir + domtree = construct_domtree(ir.cfg.blocks) + for idx in revisit # Make sure that the value we reference didn't escape - id = ir.stmts[idx][:inst].args[2].id + stmt = ir.stmts[idx][:inst]::Expr + id = (stmt.args[2]::SSAValue).id (id in relevant) || continue + #println("Revisiting ", stmt) + # We're ok to steal the memory if we don't dominate any uses ok = true - for use in uses[id] - if ssadominates(ir, domtree, idx, use) - ok = false - break + if haskey(uses, id) + for use in uses[id] + if ssadominates(ir, domtree, idx, use) + ok = false + break + end end end ok || continue - - ir.stmts[idx][:inst].args[1] = Core.mutating_arrayfreeze + stmt.args[1] = Core.mutating_arrayfreeze end + + # TODO: Use escape analysis info to determine if maybecopy should copy + + # for idx in maybecopies + # stmt = ir.stmts[idx][:inst]::Expr + # #println(stmt.args) + # arr = stmt.args[2] + # id = isa(arr, SSAValue) ? arr.id : arr.n # SSAValue or Core.Argument + + # if (id in relevant) # didn't escape elsewhere, so make a copy to keep it un-escaped + # #println("didn't escape maybecopy") + # stmt.args[1] = Main.Base.copy + # else # already escaped, so save the cost of copying and just pass the actual object + # #println("escaped maybecopy") + # ir.stmts[idx][:inst] = arr + # end + # end + return ir end diff --git a/base/pointer.jl b/base/pointer.jl index b315e589ffd9a..2c526a7063b2e 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -63,6 +63,7 @@ cconvert(::Type{Ptr{UInt8}}, s::AbstractString) = String(s) cconvert(::Type{Ptr{Int8}}, s::AbstractString) = String(s) unsafe_convert(::Type{Ptr{T}}, a::Array{T}) where {T} = ccall(:jl_array_ptr, Ptr{T}, (Any,), a) +unsafe_convert(::Type{Ptr{T}}, a::Core.ImmutableArray{T}) where {T} = ccall(:jl_array_ptr, Ptr{T}, (Any,), a) unsafe_convert(::Type{Ptr{S}}, a::AbstractArray{T}) where {S,T} = convert(Ptr{S}, unsafe_convert(Ptr{T}, a)) unsafe_convert(::Type{Ptr{T}}, a::AbstractArray{T}) where {T} = error("conversion to pointer not defined for $(typeof(a))") diff --git a/src/builtin_proto.h b/src/builtin_proto.h index f781658ed55f9..b04fc1c671b14 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -54,6 +54,7 @@ DECLARE_BUILTIN(_typevar); DECLARE_BUILTIN(arrayfreeze); DECLARE_BUILTIN(arraythaw); DECLARE_BUILTIN(mutating_arrayfreeze); +DECLARE_BUILTIN(maybecopy); JL_CALLABLE(jl_f_invoke_kwsorter); JL_CALLABLE(jl_f__structtype); diff --git a/src/builtins.c b/src/builtins.c index 7bccc23cd56df..be9fb45f2226a 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1395,6 +1395,19 @@ JL_CALLABLE(jl_f_arrayset) return args[1]; } +JL_CALLABLE(jl_f_maybecopy) +{ + // maybecopy --- this builtin is never actually supposed to be executed + // instead, calls to it are analyzed and replaced with either a call to copy + // or directly replaced with the object itself that is the target of the maybecopy + // therefore, we just check that there is one argument and do a no-op + JL_NARGSV(maybecopy, 1); + JL_TYPECHK(maybecopy, array, args[0]); + jl_array_t *a = (jl_array_t*)args[0]; + jl_array_t *na = jl_array_copy(a); + return (jl_value_t*)na; +} + // type definition ------------------------------------------------------------ JL_CALLABLE(jl_f__structtype) @@ -1877,6 +1890,7 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin_func("_setsuper!", jl_f__setsuper); jl_builtin__typebody = add_builtin_func("_typebody!", jl_f__typebody); add_builtin_func("_equiv_typedef", jl_f__equiv_typedef); + jl_builtin_maybecopy = add_builtin_func("maybecopy", jl_f_maybecopy); // builtin types add_builtin("Any", (jl_value_t*)jl_any_type); diff --git a/src/codegen.cpp b/src/codegen.cpp index 127e4a5cf8adc..bca74ca6a5fe3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -663,12 +663,13 @@ static const auto jl_newbits_func = new JuliaFunction{ // `julia.typeof` does read memory, but it is effectively readnone before we lower // the allocation function. This is OK as long as we lower `julia.typeof` no later than // `julia.gc_alloc_obj`. +// Updated to argmemonly due to C++ deconstructor style usage in jl_f_arrayfreeze / mutating_arrayfreeze static const auto jl_typeof_func = new JuliaFunction{ "julia.typeof", [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, {T_prjlvalue}, false); }, [](LLVMContext &C) { return AttributeList::get(C, - Attributes(C, {Attribute::ReadNone, Attribute::NoUnwind, Attribute::NoRecurse}), + Attributes(C, {Attribute::ArgMemOnly, Attribute::NoUnwind, Attribute::NoRecurse}), Attributes(C, {Attribute::NonNull}), None); }, }; @@ -906,6 +907,7 @@ static const std::map builtin_func_map = { { &jl_f_arrayfreeze, new JuliaFunction{"jl_f_arrayfreeze", get_func_sig, get_func_attrs} }, { &jl_f_arraythaw, new JuliaFunction{"jl_f_arraythaw", get_func_sig, get_func_attrs} }, { &jl_f_mutating_arrayfreeze,new JuliaFunction{"jl_f_mutating_arrayfreeze", get_func_sig, get_func_attrs} }, + { &jl_f_maybecopy, new JuliaFunction{"jl_f_maybecopy", get_func_sig, get_func_attrs} }, }; static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction{"jl_new_opaque_closure_jlcall", get_func_sig, get_func_attrs}; diff --git a/src/staticdata.c b/src/staticdata.c index 53e83a0aa2df9..8be7d95f2e717 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -30,7 +30,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 155 +#define NUM_TAGS 156 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -205,6 +205,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin_arrayfreeze); INSERT_TAG(jl_builtin_mutating_arrayfreeze); INSERT_TAG(jl_builtin_arraythaw); + INSERT_TAG(jl_builtin_maybecopy); // All optional tags must be placed at the end, so that we // don't accidentally have a `NULL` in the middle @@ -255,7 +256,7 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_opaque_closure_call, - &jl_f_arrayfreeze, &jl_f_mutating_arrayfreeze, &jl_f_arraythaw, + &jl_f_arrayfreeze, &jl_f_mutating_arrayfreeze, &jl_f_arraythaw, &jl_f_maybecopy, NULL }; diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 474e5dfc0f657..d4dc0d85da3a5 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,12 +1,460 @@ using Base.Experimental: ImmutableArray -function simple() +using Test + +function test_allocate1() a = Vector{Float64}(undef, 5) for i = 1:5 a[i] = i end - ImmutableArray(a) + Core.ImmutableArray(a) end + +function test_allocate2() + a = [1,2,3,4,5] + Core.ImmutableArray(a) +end + +function test_allocate3() + a = Matrix{Float64}(undef, 5, 2) + for i = 1:5 + for j = 1:2 + a[i, j] = i + j + end + end + Core.ImmutableArray(a) +end + +function test_allocate4() + a = Core.ImmutableArray{Float64}(undef, 5) +end + +function test_broadcast1() + a = Core.ImmutableArray([1,2,3]) + typeof(a .+ a) <: Core.ImmutableArray +end + +function test_allocate5() # test that throwing boundserror doesn't escape + a = [1,2,3] + try + getindex(a, 4) + catch end + Core.ImmutableArray(a) +end + +# function test_maybecopy1() +# a = Vector{Int64}(undef, 5) +# b = Core.maybecopy(a) # doesn't escape in this function - so a !=== b +# @test !(a === b) +# end + +# function test_maybecopy2() +# a = Vector{Int64}(undef, 5) +# try +# a[6] +# catch e +# @test !(e.a === a) +# end +# end + +# function test_maybecopy3() +# @noinline function escaper(arr) +# return arr +# end + +# a = Vector{Int64}(undef, 5) +# escaper(a) +# b = Core.maybecopy(a) +# @test a === b # this time, it does escape, so we give back the actual object +# end + +# function test_maybecopy4() +# @noinline function escaper(arr) +# return arr +# end + +# a = Vector{Int64}(undef, 5) +# escaper(a) +# try +# a[6] +# catch e +# if isa(e, BoundsError) +# @test e.a === a # already escaped so we don't copy +# end +# end +# end + + + + let - @allocated(simple()) - @test @allocated(simple()) < 100 + # warmup for @allocated + a,b,c,d,e = test_allocate1(), test_allocate2(), test_allocate3(), test_allocate4(), test_allocate5() + + # these magic values are ~ what the mutable array version would allocate + @test @allocated(test_allocate1()) < 100 + @test @allocated(test_allocate2()) < 100 + @test @allocated(test_allocate3()) < 150 + @test @allocated(test_allocate4()) < 100 + @test @allocated(test_allocate5()) < 170 + @test test_broadcast1() == true + + # test_maybecopy1() + # test_maybecopy2() + # test_maybecopy3() + # test_maybecopy4() end + + +# DiffEq Performance Tests + +# using DifferentialEquations +# using StaticArrays + +# function _build_atsit5_caches(::Type{T}) where {T} + +# cs = SVector{6, T}(0.161, 0.327, 0.9, 0.9800255409045097, 1.0, 1.0) + +# as = SVector{21, T}( +# #=a21=# convert(T,0.161), +# #=a31=# convert(T,-0.008480655492356989), +# #=a32=# convert(T,0.335480655492357), +# #=a41=# convert(T,2.8971530571054935), +# #=a42=# convert(T,-6.359448489975075), +# #=a43=# convert(T,4.3622954328695815), +# #=a51=# convert(T,5.325864828439257), +# #=a52=# convert(T,-11.748883564062828), +# #=a53=# convert(T,7.4955393428898365), +# #=a54=# convert(T,-0.09249506636175525), +# #=a61=# convert(T,5.86145544294642), +# #=a62=# convert(T,-12.92096931784711), +# #=a63=# convert(T,8.159367898576159), +# #=a64=# convert(T,-0.071584973281401), +# #=a65=# convert(T,-0.028269050394068383), +# #=a71=# convert(T,0.09646076681806523), +# #=a72=# convert(T,0.01), +# #=a73=# convert(T,0.4798896504144996), +# #=a74=# convert(T,1.379008574103742), +# #=a75=# convert(T,-3.290069515436081), +# #=a76=# convert(T,2.324710524099774) +# ) + +# btildes = SVector{7,T}( +# convert(T,-0.00178001105222577714), +# convert(T,-0.0008164344596567469), +# convert(T,0.007880878010261995), +# convert(T,-0.1447110071732629), +# convert(T,0.5823571654525552), +# convert(T,-0.45808210592918697), +# convert(T,0.015151515151515152) +# ) +# rs = SVector{22, T}( +# #=r11=# convert(T,1.0), +# #=r12=# convert(T,-2.763706197274826), +# #=r13=# convert(T,2.9132554618219126), +# #=r14=# convert(T,-1.0530884977290216), +# #=r22=# convert(T,0.13169999999999998), +# #=r23=# convert(T,-0.2234), +# #=r24=# convert(T,0.1017), +# #=r32=# convert(T,3.9302962368947516), +# #=r33=# convert(T,-5.941033872131505), +# #=r34=# convert(T,2.490627285651253), +# #=r42=# convert(T,-12.411077166933676), +# #=r43=# convert(T,30.33818863028232), +# #=r44=# convert(T,-16.548102889244902), +# #=r52=# convert(T,37.50931341651104), +# #=r53=# convert(T,-88.1789048947664), +# #=r54=# convert(T,47.37952196281928), +# #=r62=# convert(T,-27.896526289197286), +# #=r63=# convert(T,65.09189467479366), +# #=r64=# convert(T,-34.87065786149661), +# #=r72=# convert(T,1.5), +# #=r73=# convert(T,-4), +# #=r74=# convert(T,2.5), +# ) +# return cs, as, btildes, rs +# end + +# function test_imarrays() +# function lorenz(u, p, t) +# a,b,c = u +# x,y,z = p +# dx_dt = x * (b - a) +# dy_dt = a*(y - c) - b +# dz_dt = a*b - z * c +# res = Vector{Float64}(undef, 3) +# res[1], res[2], res[3] = dx_dt, dy_dt, dz_dt +# Core.ImmutableArray(res) +# end + +# _u0 = Core.ImmutableArray([1.0, 1.0, 1.0]) +# _tspan = (0.0, 100.0) +# _p = (10.0, 28.0, 8.0/3.0) +# prob = ODEProblem(lorenz, _u0, _tspan, _p) + +# u0 = prob.u0 +# tspan = prob.tspan +# f = prob.f +# p = prob.p + +# dt = 0.1f0 +# saveat = nothing +# save_everystep = true +# abstol = 1f-6 +# reltol = 1f-3 + +# t = tspan[1] +# tf = prob.tspan[2] + +# beta1 = 7/50 +# beta2 = 2/25 +# qmax = 10.0 +# qmin = 1/5 +# gamma = 9/10 +# qoldinit = 1e-4 + +# if saveat === nothing +# ts = Vector{eltype(dt)}(undef,1) +# ts[1] = prob.tspan[1] +# us = Vector{typeof(u0)}(undef,0) +# push!(us,recursivecopy(u0)) +# else +# ts = saveat +# cur_t = 1 +# us = MVector{length(ts),typeof(u0)}(undef) +# if prob.tspan[1] == ts[1] +# cur_t += 1 +# us[1] = u0 +# end +# end + +# u = u0 +# qold = 1e-4 +# k7 = f(u, p, t) + +# cs, as, btildes, rs = _build_atsit5_caches(eltype(u0)) +# c1, c2, c3, c4, c5, c6 = cs +# a21, a31, a32, a41, a42, a43, a51, a52, a53, a54, +# a61, a62, a63, a64, a65, a71, a72, a73, a74, a75, a76 = as +# btilde1, btilde2, btilde3, btilde4, btilde5, btilde6, btilde7 = btildes + +# # FSAL +# while t < tspan[2] +# uprev = u +# k1 = k7 +# EEst = Inf + +# while EEst > 1 +# dt < 1e-14 && error("dt 1 +# dt = dt/min(inv(qmin),q11/gamma) +# else # EEst <= 1 +# @fastmath q = max(inv(qmax),min(inv(qmin),q/gamma)) +# qold = max(EEst,qoldinit) +# dtold = dt +# dt = dt/q #dtnew +# dt = min(abs(dt),abs(tf-t-dtold)) +# told = t + +# if (tf - t - dtold) < 1e-14 +# t = tf +# else +# t += dtold +# end + +# if saveat === nothing && save_everystep +# push!(us,recursivecopy(u)) +# push!(ts,t) +# else saveat !== nothing +# while cur_t <= length(ts) && ts[cur_t] <= t +# savet = ts[cur_t] +# θ = (savet - told)/dtold +# b1θ, b2θ, b3θ, b4θ, b5θ, b6θ, b7θ = bθs(rs, θ) +# us[cur_t] = uprev + dtold*( +# b1θ*k1 + b2θ*k2 + b3θ*k3 + b4θ*k4 + b5θ*k5 + b6θ*k6 + b7θ*k7) +# cur_t += 1 +# end +# end +# end +# end +# end + +# if saveat === nothing && !save_everystep +# push!(us,u) +# push!(ts,t) +# end + +# sol = DiffEqBase.build_solution(prob,Tsit5(),ts,us,calculate_error = false) + +# DiffEqBase.has_analytic(prob.f) && DiffEqBase.calculate_solution_errors!(sol;timeseries_errors=true,dense_errors=false) + +# sol +# end + +# function test_marrays() +# function lorenz(u, p, t) +# a,b,c = u +# x,y,z = p +# dx_dt = x * (b - a) +# dy_dt = a*(y - c) - b +# dz_dt = a*b - z * c +# res = Vector{Float64}(undef, 3) +# res[1], res[2], res[3] = dx_dt, dy_dt, dz_dt +# res +# end + +# _u0 = [1.0, 1.0, 1.0] +# _tspan = (0.0, 100.0) +# _p = (10.0, 28.0, 8.0/3.0) +# prob = ODEProblem(lorenz, _u0, _tspan, _p) + +# u0 = prob.u0 +# tspan = prob.tspan +# f = prob.f +# p = prob.p + +# dt = 0.1f0 +# saveat = nothing +# save_everystep = true +# abstol = 1f-6 +# reltol = 1f-3 + +# t = tspan[1] +# tf = prob.tspan[2] + +# beta1 = 7/50 +# beta2 = 2/25 +# qmax = 10.0 +# qmin = 1/5 +# gamma = 9/10 +# qoldinit = 1e-4 + +# if saveat === nothing +# ts = Vector{eltype(dt)}(undef,1) +# ts[1] = prob.tspan[1] +# us = Vector{typeof(u0)}(undef,0) +# push!(us,recursivecopy(u0)) +# else +# ts = saveat +# cur_t = 1 +# us = MVector{length(ts),typeof(u0)}(undef) +# if prob.tspan[1] == ts[1] +# cur_t += 1 +# us[1] = u0 +# end +# end + +# u = u0 +# qold = 1e-4 +# k7 = f(u, p, t) + +# cs, as, btildes, rs = _build_atsit5_caches(eltype(u0)) +# c1, c2, c3, c4, c5, c6 = cs +# a21, a31, a32, a41, a42, a43, a51, a52, a53, a54, +# a61, a62, a63, a64, a65, a71, a72, a73, a74, a75, a76 = as +# btilde1, btilde2, btilde3, btilde4, btilde5, btilde6, btilde7 = btildes + +# # FSAL +# while t < tspan[2] +# uprev = u +# k1 = k7 +# EEst = Inf + +# while EEst > 1 +# dt < 1e-14 && error("dt 1 +# dt = dt/min(inv(qmin),q11/gamma) +# else # EEst <= 1 +# @fastmath q = max(inv(qmax),min(inv(qmin),q/gamma)) +# qold = max(EEst,qoldinit) +# dtold = dt +# dt = dt/q #dtnew +# dt = min(abs(dt),abs(tf-t-dtold)) +# told = t + +# if (tf - t - dtold) < 1e-14 +# t = tf +# else +# t += dtold +# end + +# if saveat === nothing && save_everystep +# push!(us,recursivecopy(u)) +# push!(ts,t) +# else saveat !== nothing +# while cur_t <= length(ts) && ts[cur_t] <= t +# savet = ts[cur_t] +# θ = (savet - told)/dtold +# b1θ, b2θ, b3θ, b4θ, b5θ, b6θ, b7θ = bθs(rs, θ) +# us[cur_t] = uprev + dtold*( +# b1θ*k1 + b2θ*k2 + b3θ*k3 + b4θ*k4 + b5θ*k5 + b6θ*k6 + b7θ*k7) +# cur_t += 1 +# end +# end +# end +# end +# end + +# if saveat === nothing && !save_everystep +# push!(us,u) +# push!(ts,t) +# end + +# sol = DiffEqBase.build_solution(prob,Tsit5(),ts,us,calculate_error = false) + +# DiffEqBase.has_analytic(prob.f) && DiffEqBase.calculate_solution_errors!(sol;timeseries_errors=true,dense_errors=false) + +# sol +# end + From 424232ab7f4c138d60022512ab3d359c9916f7a0 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Tue, 9 Nov 2021 18:43:35 -0500 Subject: [PATCH 03/41] Begin EscapeAnalysis.jl port --- base/array.jl | 26 ++- base/compiler/optimize.jl | 399 ++++++++++++++++++++++++++++++++++ base/compiler/ssair/passes.jl | 53 ++--- base/compiler/types.jl | 2 +- test/core.jl | 3 +- 5 files changed, 447 insertions(+), 36 deletions(-) diff --git a/base/array.jl b/base/array.jl index dc22cc4a21cb0..4fb35cb7540bb 100644 --- a/base/array.jl +++ b/base/array.jl @@ -252,19 +252,26 @@ length(a::Array) = arraylen(a) elsize(::Type{<:Array{T}}) where {T} = aligned_sizeof(T) sizeof(a::Array) = Core.sizeof(a) -function isassigned(a::Array, i::Int...) +function isassigned(a::IMArray, i::Int...) @_inline_meta ii = (_sub2ind(size(a), i...) % UInt) - 1 @boundscheck ii < length(a) % UInt || return false ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 end -function isassigned(a::ImmutableArray, i::Int...) - @_inline_meta - ii = (_sub2ind(size(a), i...) % UInt) - 1 - @boundscheck ii < length(a) % UInt || return false - ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 -end +# function isassigned(a::Array, i::Int...) +# @_inline_meta +# ii = (_sub2ind(size(a), i...) % UInt) - 1 +# @boundscheck ii < length(a) % UInt || return false +# ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 +# end + +# function isassigned(a::ImmutableArray, i::Int...) +# @_inline_meta +# ii = (_sub2ind(size(a), i...) % UInt) - 1 +# @boundscheck ii < length(a) % UInt || return false +# ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 +# end ## copy ## @@ -431,7 +438,7 @@ ImmutableArray{T}(undef::UndefInitializer, dims::Dims) where T = ImmutableArray( To do so, the optimizer decides whether to create a copy of `x` or not based on the implementation That is, `maybecopy` will either be a call to [`copy`](@ref) or just a reference to x. """ -maybecopy = Core.maybecopy +const maybecopy = Core.maybecopy # T[x...] constructs Array{T,1} """ @@ -954,6 +961,9 @@ function getindex end @eval getindex(A::ImmutableArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) @eval getindex(A::ImmutableArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) +# @eval getindex(A::IMArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) +# @eval getindex(A::IMArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) + # Faster contiguous indexing using copyto! for UnitRange and Colon function getindex(A::Array, I::AbstractUnitRange{<:Integer}) @_inline_meta diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index de6294393d3c1..b2692711ff956 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -308,6 +308,11 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) #@Base.show ("after_adce", ir) @timeit "type lift" ir = type_lift_pass!(ir) #@timeit "compact 3" ir = compact!(ir) + nargs = let def = sv.linfo.def + isa(def, Method) ? Int(def.nargs) : 0 + end + esc_state = find_escapes(ir, nargs) + @eval Main (esc = $esc_state) ir = memory_opt!(ir) #@Base.show ir if JLOptions().debug_level == 2 @@ -667,3 +672,397 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab end end end + +struct EscapeLattice + Analyzed::Bool + ReturnEscape::BitSet + ThrownEscape::Bool + GlobalEscape::Bool + # TODO: ArgEscape::Int +end + +function (==)(x::EscapeLattice, y::EscapeLattice) + return x.Analyzed === y.Analyzed && + x.ReturnEscape == y.ReturnEscape && + x.ThrownEscape === y.ThrownEscape && + x.GlobalEscape === y.GlobalEscape +end + +const NO_RETURN = BitSet() +const ARGUMENT_RETURN = BitSet(0) +NotAnalyzed() = EscapeLattice(false, NO_RETURN, false, false) # not formally part of the lattice +NoEscape() = EscapeLattice(true, NO_RETURN, false, false) +ReturnEscape(pcs::BitSet) = EscapeLattice(true, pcs, false, false) +ReturnEscape(pc::Int) = ReturnEscape(BitSet(pc)) +ArgumentReturnEscape() = ReturnEscape(ARGUMENT_RETURN) +ThrownEscape() = EscapeLattice(true, NO_RETURN, true, false) +GlobalEscape() = EscapeLattice(true, NO_RETURN, false, true) +let + all_return = BitSet(0:100_000) + global AllReturnEscape() = ReturnEscape(all_return) # used for `show` + global AllEscape() = EscapeLattice(true, all_return, true, true) +end + +has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() +has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() +has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) +has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape +has_thrown_escape(x::EscapeLattice) = x.ThrownEscape +has_global_escape(x::EscapeLattice) = x.GlobalEscape +has_all_escape(x::EscapeLattice) = AllEscape() == x + +function ⊑(x::EscapeLattice, y::EscapeLattice) + if x.Analyzed ≤ y.Analyzed && + x.ReturnEscape ⊆ y.ReturnEscape && + x.ThrownEscape ≤ y.ThrownEscape && + x.GlobalEscape ≤ y.GlobalEscape + return true + end + return false +end + +⋤(x::EscapeLattice, y::EscapeLattice) = ⊑(x, y) && !⊑(y, x) + +function ⊔(x::EscapeLattice, y::EscapeLattice) + return EscapeLattice( + x.Analyzed | y.Analyzed, + x.ReturnEscape ∪ y.ReturnEscape, + x.ThrownEscape | y.ThrownEscape, + x.GlobalEscape | y.GlobalEscape, + ) +end + +function ⊓(x::EscapeLattice, y::EscapeLattice) + return EscapeLattice( + x.Analyzed & y.Analyzed, + x.ReturnEscape ∩ y.ReturnEscape, + x.ThrownEscape & y.ThrownEscape, + x.GlobalEscape & y.GlobalEscape, + ) +end + +struct EscapeState + arguments::Vector{EscapeLattice} + ssavalues::Vector{EscapeLattice} +end + +function EscapeState(nslots::Int, nargs::Int, nstmts::Int) + arguments = EscapeLattice[ + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] + ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] + return EscapeState(arguments, ssavalues) +end + +const Change = Pair{Union{Argument,SSAValue},EscapeLattice} +const Changes = Vector{Change} + +function propagate_changes!(state::EscapeState, changes::Changes) + local anychanged = false + + for (x, info) in changes + if isa(x, Argument) + old = state.arguments[x.n] + new = old ⊔ info + if old ≠ new + state.arguments[x.n] = new + anychanged |= true + end + else + x = x::SSAValue + old = state.ssavalues[x.id] + new = old ⊔ info + if old ≠ new + state.ssavalues[x.id] = new + anychanged |= true + end + end + end + + return anychanged +end + +# function normalize(@nospecialize(x)) +# if isa(x, QuoteNode) +# return x.value +# else +# return x +# end +# end + +function add_changes!(args::Vector{Any}, ir::IRCode, info::EscapeLattice, changes::Changes) + for x in args + add_change!(x, ir, info, changes) + end +end + +function add_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) + if isa(x, Argument) || isa(x, SSAValue) + if !isbitstype(widenconst(argextype(x, ir, ir.sptypes, ir.argtypes))) + push!(changes, Change(x, info)) + end + end +end + +function escape_invoke!(args::Vector{Any}, pc::Int, + state::EscapeState, ir::IRCode, changes::Changes) + linfo = first(args)::MethodInstance + cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) + args = args[2:end] + if isnothing(cache) + add_changes!(args, ir, AllEscape(), changes) + else + (linfostate, _ #=ir::IRCode=#) = cache + retinfo = state.ssavalues[pc] # escape information imposed on the call statement + method = linfo.def::Method + nargs = Int(method.nargs) + for i in 1:length(args) + arg = args[i] + if i ≤ nargs + arginfo = linfostate.arguments[i] + else # handle isva signature: COMBAK will this invalid once we encode alias information ? + arginfo = linfostate.arguments[nargs] + end + if isempty(arginfo.ReturnEscape) + @eval Main (ir = $ir; linfo = $linfo) + error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") + end + info = from_interprocedural(arginfo, retinfo) + add_change!(arg, ir, info, changes) + end + end +end + +# reinterpret the escape information imposed on the callee argument (`arginfo`) in the +# context of the caller frame using the escape information imposed on the return value (`retinfo`) +function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice) + ar = arginfo.ReturnEscape + newarginfo = EscapeLattice(true, NO_RETURN, arginfo.ThrownEscape, arginfo.GlobalEscape) + if ar == ARGUMENT_RETURN + # if this is simply passed as the call argument, we can discard the `ReturnEscape` + # information and just propagate the other escape information + return newarginfo + else + # if this can be a return value, we have to merge it with the escape information + return newarginfo ⊔ retinfo + end +end + +function escape_call!(args::Vector{Any}, pc::Int, + state::EscapeState, ir::IRCode, changes::Changes) + ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) + f = singleton_type(ft) + if isa(f, Core.IntrinsicFunction) + return false # COMBAK we may break soundness here, e.g. `pointerref` + end + ishandled = escape_builtin!(f, args, pc, state, ir, changes)::Union{Nothing,Bool} + Main.Base.isnothing(ishandled) && return false # nothing to propagate + if !ishandled + # if this call hasn't been handled by any of pre-defined handlers, + # we escape this call conservatively + add_changes!(args[2:end], ir, AllEscape(), changes) + end + return true +end + +# TODO implement more builtins, make them more accurate +# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? + +escape_builtin!(@nospecialize(f), _...) = return false + +escape_builtin!(::typeof(isa), _...) = return nothing +escape_builtin!(::typeof(typeof), _...) = return nothing +escape_builtin!(::typeof(Core.sizeof), _...) = return nothing +escape_builtin!(::typeof(===), _...) = return nothing + +function escape_builtin!(::typeof(Core.ifelse), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) + length(args) == 4 || return false + f, cond, th, el = args + info = state.ssavalues[pc] + condt = argextype(cond, ir, ir.sptypes, ir.argtypes) + if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) + if cond + add_change!(th, ir, info, changes) + else + add_change!(el, ir, info, changes) + end + else + add_change!(th, ir, info, changes) + add_change!(el, ir, info, changes) + end + return true +end + +function escape_builtin!(::typeof(typeassert), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) + length(args) == 3 || return false + f, obj, typ = args + info = state.ssavalues[pc] + add_change!(obj, ir, info, changes) + return true +end + +function escape_builtin!(::typeof(tuple), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + end + add_changes!(args[2:end], ir, info, changes) + return true +end + +# TODO don't propagate escape information to the 1st argument, but propagate information to aliased field +function escape_builtin!(::typeof(getfield), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + end + # only propagate info when the field itself is non-bitstype + if !isbitstype(widenconst(ir.stmts.type[pc])) + add_changes!(args[2:end], ir, info, changes) + end + return true +end + +function find_escapes(ir::IRCode, nargs::Int) + (; stmts, sptypes, argtypes) = ir + nstmts = length(stmts) + + # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties + state = EscapeState(length(ir.argtypes), nargs, nstmts) + changes = Changes() # stashes changes that happen at current statement + + while true + local anyupdate = false + + for pc in nstmts:-1:1 + stmt = stmts.inst[pc] + + # we escape statements with the `ThrownEscape` property using the effect-freeness + # information computed by the inliner + is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 + + # collect escape information + if isa(stmt, Expr) + head = stmt.head + if head === :call + has_changes = escape_call!(stmt.args, pc, state, ir, changes) + if !is_effect_free + add_changes!(stmt.args, ir, ThrownEscape(), changes) + else + has_changes || continue + end + elseif head === :invoke + escape_invoke!(stmt.args, pc, state, ir, changes) + elseif head === :new + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not + end + add_changes!(stmt.args[2:end], ir, info, changes) + elseif head === :splatnew + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not + end + # splatnew passes field values using a single tuple (args[2]) + add_change!(stmt.args[2], ir, info, changes) + elseif head === :(=) + lhs, rhs = stmt.args + if isa(lhs, GlobalRef) # global store + add_change!(rhs, ir, GlobalEscape(), changes) + end + elseif head === :foreigncall + # for foreigncall we simply escape every argument (args[6:length(args[3])]) + # and its name (args[1]) + # TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls + foreigncall_nargs = length((stmt.args[3])::SimpleVector) + name = stmt.args[1] + # if normalize(name) === :jl_gc_add_finalizer_th + # continue # XXX assume this finalizer call is valid for finalizer elision + # end + add_change!(name, ir, ThrownEscape(), changes) + add_changes!(stmt.args[6:5+foreigncall_nargs], ir, ThrownEscape(), changes) + elseif head === :throw_undef_if_not # XXX when is this expression inserted ? + add_change!(stmt.args[1], ir, ThrownEscape(), changes) + elseif is_meta_expr_head(head) + # meta expressions doesn't account for any usages + continue + elseif head === :static_parameter + # :static_parameter refers any of static parameters, but since they exist + # statically, we're really not interested in their escapes + continue + elseif head === :copyast + # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values + continue + elseif head === :undefcheck + # undefcheck is temporarily inserted by compiler + # it will be processd be later pass so it won't change any of escape states + continue + elseif head === :the_exception + # we don't propagate escape information on exceptions via this expression, but rather + # use a dedicated lattice property `ThrownEscape` + continue + elseif head === :isdefined + # just returns `Bool`, nothing accounts for any usages + continue + elseif head === :enter || head === :leave || head === :pop_exception + # these exception frame managements doesn't account for any usages + # we can just ignore escape information from + continue + elseif head === :gc_preserve_begin || head === :gc_preserve_end + # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information + # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere + continue + else + add_changes!(stmt.args, ir, AllEscape(), changes) + end + elseif isa(stmt, GlobalRef) # global load + add_change!(SSAValue(pc), ir, GlobalEscape(), changes) + elseif isa(stmt, PiNode) + if isdefined(stmt, :val) + info = state.ssavalues[pc] + add_change!(stmt.val, ir, info, changes) + end + elseif isa(stmt, PhiNode) + info = state.ssavalues[pc] + values = stmt.values + for i in 1:length(values) + if isassigned(values, i) + add_change!(values[i], ir, info, changes) + end + end + elseif isa(stmt, PhiCNode) + info = state.ssavalues[pc] + values = stmt.values + for i in 1:length(values) + if isassigned(values, i) + add_change!(values[i], ir, info, changes) + end + end + elseif isa(stmt, UpsilonNode) + if isdefined(stmt, :val) + info = state.ssavalues[pc] + add_change!(stmt.val, ir, info, changes) + end + elseif isa(stmt, ReturnNode) + if isdefined(stmt, :val) + add_change!(stmt.val, ir, ReturnEscape(pc), changes) + end + else + #@assert stmt isa GotoNode || stmt isa GotoIfNot || stmt isa GlobalRef || isnothing(stmt) + continue + end + + isempty(changes) && continue + + anyupdate |= propagate_changes!(state, changes) + + empty!(changes) + end + + anyupdate || break + end + + return state +end \ No newline at end of file diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 723776f49fe37..551c146e4331d 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1256,29 +1256,29 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end -# function is_known_fcall(stmt::Expr, @nospecialize(func)) -# isexpr(stmt, :foreigncall) || return false -# s = stmt.args[1] -# isa(s, QuoteNode) && (s = s.value) -# return s === func -# end - -function is_known_fcall(stmt::Expr, funcs::Vector{Symbol}) +function is_known_fcall(stmt::Expr, funcs) isexpr(stmt, :foreigncall) || return false s = stmt.args[1] isa(s, QuoteNode) && (s = s.value) - # return any(e -> s === e, funcs) - return true in map(e -> s === e, funcs) + isa(s, Symbol) || return false + for func in funcs + s === func && return true + end + return false end -function is_allocation(stmt::Expr) - isexpr(stmt, :foreigncall) || return false - s = stmt.args[1] - isa(s, QuoteNode) && (s = s.value) - return (s === :jl_alloc_array_1d - || s === :jl_alloc_array_2d - || s === :jl_alloc_array_3d - || s === :jl_new_array) +is_array_allocation(stmt::Expr) = + is_known_fcall(stmt, + (:jl_alloc_array_1d, + :jl_alloc_array_2d, + :jl_alloc_array_3d, + :jl_new_array)) + +function memory_opt!(ir::IRCode, state) + compact = IncrementalCompact(ir, false) + ir = finish(compact) + println("analyzed escapes : ", state) + return ir end function memory_opt!(ir::IRCode) @@ -1339,7 +1339,7 @@ function memory_opt!(ir::IRCode) continue end - if is_allocation(stmt) + if is_array_allocation(stmt) push!(relevant, idx) # TODO: Mark everything else here continue @@ -1355,13 +1355,13 @@ function memory_opt!(ir::IRCode) arr = stmt.args[3] mark_use(arr, idx) - elseif is_known_call(stmt, setindex!, compact) && length(stmt.args) == 4 - # handle similarly to arrayset - val = stmt.args[3] - mark_escape(val) + # elseif is_known_call(stmt, setindex!, compact) && length(stmt.args) == 4 + # # handle similarly to arrayset + # val = stmt.args[3] + # mark_escape(val) - arr = stmt.args[2] - mark_use(arr, idx) + # arr = stmt.args[2] + # mark_use(arr, idx) elseif is_known_call(stmt, (===), compact) && length(stmt.args) == 3 arr1 = stmt.args[2] @@ -1371,7 +1371,7 @@ function memory_opt!(ir::IRCode) mark_use(arr2, idx) # these foreigncalls have similar structure and don't escape our array, so handle them all at once - elseif is_known_fcall(stmt, [:jl_array_ptr, :jl_array_copy]) && length(stmt.args) == 6 + elseif is_known_fcall(stmt, (:jl_array_ptr, :jl_array_copy)) && length(stmt.args) == 6 arr = stmt.args[6] mark_use(arr, idx) @@ -1399,6 +1399,7 @@ function memory_opt!(ir::IRCode) for idx in revisit # Make sure that the value we reference didn't escape stmt = ir.stmts[idx][:inst]::Expr + #println("test, stmt : ", stmt) id = (stmt.args[2]::SSAValue).id (id in relevant) || continue diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 1a89d5e994b15..e439c3c003892 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -162,7 +162,7 @@ struct NativeInterpreter <: AbstractInterpreter end # If they didn't pass typemax(UInt) but passed something more subtly - # incorrect, fail out loudly. + # incorrect, fail out loud. @assert world <= get_world_counter() diff --git a/test/core.jl b/test/core.jl index a9c9f6222cd9f..37c1dc3a91381 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2557,7 +2557,8 @@ end # pull request #9534 @test_throws BoundsError((1, 2), 3) begin; a, b, c = 1, 2; end let a = [] - @test try; a[]; catch ex; (ex::BoundsError).a === a && ex.i == (); end + # no longer passes because BoundsError now copies arrays + # @test try; a[]; catch ex; (ex::BoundsError).a === a && ex.i == (); end @test_throws BoundsError(a, (1, 2)) a[1, 2] @test_throws BoundsError(a, (10,)) a[10] end From e031da8915cbcec2dbdef2ce7f092c2fcb820277 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Mon, 20 Dec 2021 20:17:16 -0500 Subject: [PATCH 04/41] Update memory_opt! for usage with EscapeState, begin port of EA.jl into Core --- base/compiler/optimize.jl | 195 ++++++++++++++++++-------------- base/compiler/ssair/passes.jl | 151 ++++--------------------- test/compiler/immutablearray.jl | 1 - 3 files changed, 130 insertions(+), 217 deletions(-) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index b2692711ff956..e126b742f1f39 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -113,6 +113,93 @@ function ir_to_codeinf!(opt::OptimizationState) end end +################## +# EscapeAnalysis # +################## + +struct EscapeLattice + Analyzed::Bool + ReturnEscape + ThrownEscape::Bool + GlobalEscape::Bool + # TODO: ArgEscape::Int +end + +function (==)(x::EscapeLattice, y::EscapeLattice) + return x.Analyzed === y.Analyzed && + x.ReturnEscape == y.ReturnEscape && + x.ThrownEscape === y.ThrownEscape && + x.GlobalEscape === y.GlobalEscape +end + +const NO_RETURN = BitSet() +const ARGUMENT_RETURN = BitSet(0) +NotAnalyzed() = EscapeLattice(false, NO_RETURN, false, false) # not formally part of the lattice +NoEscape() = EscapeLattice(true, NO_RETURN, false, false) +ReturnEscape(pcs::BitSet) = EscapeLattice(true, pcs, false, false) +ReturnEscape(pc::Int) = ReturnEscape(BitSet(pc)) +ArgumentReturnEscape() = ReturnEscape(ARGUMENT_RETURN) +ThrownEscape() = EscapeLattice(true, NO_RETURN, true, false) +GlobalEscape() = EscapeLattice(true, NO_RETURN, false, true) +let + all_return = BitSet(0:100_000) + global AllReturnEscape() = ReturnEscape(all_return) # used for `show` + global AllEscape() = EscapeLattice(true, all_return, true, true) +end + +function ⊑(x::EscapeLattice, y::EscapeLattice) + if x.Analyzed ≤ y.Analyzed && + x.ReturnEscape ⊆ y.ReturnEscape && + x.ThrownEscape ≤ y.ThrownEscape && + x.GlobalEscape ≤ y.GlobalEscape + return true + end + return false +end + +⋤(x::EscapeLattice, y::EscapeLattice) = ⊑(x, y) && !⊑(y, x) + +function ⊔(x::EscapeLattice, y::EscapeLattice) + return EscapeLattice( + x.Analyzed | y.Analyzed, + x.ReturnEscape ∪ y.ReturnEscape, + x.ThrownEscape | y.ThrownEscape, + x.GlobalEscape | y.GlobalEscape, + ) +end + +function ⊓(x::EscapeLattice, y::EscapeLattice) + return EscapeLattice( + x.Analyzed & y.Analyzed, + x.ReturnEscape ∩ y.ReturnEscape, + x.ThrownEscape & y.ThrownEscape, + x.GlobalEscape & y.GlobalEscape, + ) +end + +has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() +has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() +has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) +has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape +has_thrown_escape(x::EscapeLattice) = x.ThrownEscape +has_global_escape(x::EscapeLattice) = x.GlobalEscape +has_all_escape(x::EscapeLattice) = AllEscape() == x + +const Change = Pair{Union{Argument,SSAValue},EscapeLattice} +const Changes = Vector{Change} + +struct EscapeState + arguments::Vector{EscapeLattice} + ssavalues::Vector{EscapeLattice} +end + +function EscapeState(nslots::Int, nargs::Int, nstmts::Int) + arguments = EscapeLattice[ + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] + ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] + return EscapeState(arguments, ssavalues) +end + ############# # constants # ############# @@ -145,6 +232,9 @@ const _PURE_OR_ERROR_BUILTINS = [ const TOP_TUPLE = GlobalRef(Core, :tuple) +const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance, EscapeState}() +__clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) + ######### # logic # ######### @@ -312,8 +402,9 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) isa(def, Method) ? Int(def.nargs) : 0 end esc_state = find_escapes(ir, nargs) - @eval Main (esc = $esc_state) - ir = memory_opt!(ir) + setindex!(GLOBAL_ESCAPE_CACHE, esc_state, sv.linfo) + # @eval Main (esc = $esc_state) + ir = memory_opt!(ir, esc_state) #@Base.show ir if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) @@ -591,6 +682,17 @@ function is_known_call(e::Expr, @nospecialize(func), src, sptypes::Vector{Any}, return isa(f, Const) && f.val === func end +function is_known_fcall(stmt::Expr, funcs) + isexpr(stmt, :foreigncall) || return false + s = stmt.args[1] + isa(s, QuoteNode) && (s = s.value) + isa(s, Symbol) || return false + for func in funcs + s === func && return true + end + return false +end + function renumber_ir_elements!(body::Vector{Any}, changemap::Vector{Int}) return renumber_ir_elements!(body, changemap, changemap) end @@ -673,89 +775,6 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab end end -struct EscapeLattice - Analyzed::Bool - ReturnEscape::BitSet - ThrownEscape::Bool - GlobalEscape::Bool - # TODO: ArgEscape::Int -end - -function (==)(x::EscapeLattice, y::EscapeLattice) - return x.Analyzed === y.Analyzed && - x.ReturnEscape == y.ReturnEscape && - x.ThrownEscape === y.ThrownEscape && - x.GlobalEscape === y.GlobalEscape -end - -const NO_RETURN = BitSet() -const ARGUMENT_RETURN = BitSet(0) -NotAnalyzed() = EscapeLattice(false, NO_RETURN, false, false) # not formally part of the lattice -NoEscape() = EscapeLattice(true, NO_RETURN, false, false) -ReturnEscape(pcs::BitSet) = EscapeLattice(true, pcs, false, false) -ReturnEscape(pc::Int) = ReturnEscape(BitSet(pc)) -ArgumentReturnEscape() = ReturnEscape(ARGUMENT_RETURN) -ThrownEscape() = EscapeLattice(true, NO_RETURN, true, false) -GlobalEscape() = EscapeLattice(true, NO_RETURN, false, true) -let - all_return = BitSet(0:100_000) - global AllReturnEscape() = ReturnEscape(all_return) # used for `show` - global AllEscape() = EscapeLattice(true, all_return, true, true) -end - -has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() -has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() -has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) -has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape -has_thrown_escape(x::EscapeLattice) = x.ThrownEscape -has_global_escape(x::EscapeLattice) = x.GlobalEscape -has_all_escape(x::EscapeLattice) = AllEscape() == x - -function ⊑(x::EscapeLattice, y::EscapeLattice) - if x.Analyzed ≤ y.Analyzed && - x.ReturnEscape ⊆ y.ReturnEscape && - x.ThrownEscape ≤ y.ThrownEscape && - x.GlobalEscape ≤ y.GlobalEscape - return true - end - return false -end - -⋤(x::EscapeLattice, y::EscapeLattice) = ⊑(x, y) && !⊑(y, x) - -function ⊔(x::EscapeLattice, y::EscapeLattice) - return EscapeLattice( - x.Analyzed | y.Analyzed, - x.ReturnEscape ∪ y.ReturnEscape, - x.ThrownEscape | y.ThrownEscape, - x.GlobalEscape | y.GlobalEscape, - ) -end - -function ⊓(x::EscapeLattice, y::EscapeLattice) - return EscapeLattice( - x.Analyzed & y.Analyzed, - x.ReturnEscape ∩ y.ReturnEscape, - x.ThrownEscape & y.ThrownEscape, - x.GlobalEscape & y.GlobalEscape, - ) -end - -struct EscapeState - arguments::Vector{EscapeLattice} - ssavalues::Vector{EscapeLattice} -end - -function EscapeState(nslots::Int, nargs::Int, nstmts::Int) - arguments = EscapeLattice[ - 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] - ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] - return EscapeState(arguments, ssavalues) -end - -const Change = Pair{Union{Argument,SSAValue},EscapeLattice} -const Changes = Vector{Change} - function propagate_changes!(state::EscapeState, changes::Changes) local anychanged = false @@ -808,10 +827,10 @@ function escape_invoke!(args::Vector{Any}, pc::Int, linfo = first(args)::MethodInstance cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) args = args[2:end] - if isnothing(cache) + if cache === nothing add_changes!(args, ir, AllEscape(), changes) else - (linfostate, _ #=ir::IRCode=#) = cache + linfostate = cache retinfo = state.ssavalues[pc] # escape information imposed on the call statement method = linfo.def::Method nargs = Int(method.nargs) @@ -855,7 +874,7 @@ function escape_call!(args::Vector{Any}, pc::Int, return false # COMBAK we may break soundness here, e.g. `pointerref` end ishandled = escape_builtin!(f, args, pc, state, ir, changes)::Union{Nothing,Bool} - Main.Base.isnothing(ishandled) && return false # nothing to propagate + ishandled === nothing && return false # nothing to propagate if !ishandled # if this call hasn't been handled by any of pre-defined handlers, # we escape this call conservatively diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 551c146e4331d..60cb7811d6574 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1256,138 +1256,42 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end -function is_known_fcall(stmt::Expr, funcs) - isexpr(stmt, :foreigncall) || return false - s = stmt.args[1] - isa(s, QuoteNode) && (s = s.value) - isa(s, Symbol) || return false - for func in funcs - s === func && return true - end - return false -end - -is_array_allocation(stmt::Expr) = - is_known_fcall(stmt, +is_array_allocation(stmt::Expr) = + is_known_fcall(stmt, (:jl_alloc_array_1d, :jl_alloc_array_2d, :jl_alloc_array_3d, :jl_new_array)) -function memory_opt!(ir::IRCode, state) - compact = IncrementalCompact(ir, false) - ir = finish(compact) - println("analyzed escapes : ", state) - return ir -end - -function memory_opt!(ir::IRCode) +function memory_opt!(ir::IRCode, escape_state) compact = IncrementalCompact(ir, false) - uses = IdDict{Int, Vector{Int}}() - relevant = IdSet{Int}() # allocations + # relevant = IdSet{Int}() # allocations revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in maybecopies = Int[] # calls to maybecopy - function mark_escape(@nospecialize val) - isa(val, SSAValue) || return - #println(val.id, " escaped.") - val.id in relevant && pop!(relevant, val.id) - end + # function mark_escape(@nospecialize val) + # isa(val, SSAValue) || return + # #println(val.id, " escaped.") + # val.id in relevant && pop!(relevant, val.id) + # end - function mark_use(val, idx) - isa(val, SSAValue) || return - id = val.id - id in relevant || return - (haskey(uses, id)) || (uses[id] = Int[]) - push!(uses[id], idx) - end + # function mark_use(val, idx) + # isa(val, SSAValue) || return + # id = val.id + # id in relevant || return + # (haskey(uses, id)) || (uses[id] = Int[]) + # push!(uses[id], idx) + # end for ((_, idx), stmt) in compact - - #println("idx: ", idx, " = ", stmt) - - if isa(stmt, ReturnNode) - isdefined(stmt, :val) || continue - val = stmt.val - mark_use(val, idx) - continue - - # check for phinodes that are possibly allocations - elseif isa(stmt, PhiNode) - - # ensure all of the phinode values are defined - defined = true - for i = 1:length(stmt.values) - if !isassigned(stmt.values, i) - defined = false - end - end - - defined || continue - - for val in stmt.values - if isa(val, SSAValue) && val.id in relevant - push!(relevant, idx) - end - end - end - (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue if is_known_call(stmt, Core.maybecopy, compact) push!(maybecopies, idx) - continue - end - - if is_array_allocation(stmt) - push!(relevant, idx) - # TODO: Mark everything else here - continue - end - - if is_known_call(stmt, arrayset, compact) && length(stmt.args) >= 5 - # The value being set escapes, everything else doesn't - mark_escape(stmt.args[4]) - arr = stmt.args[3] - mark_use(arr, idx) - - elseif is_known_call(stmt, arrayref, compact) && length(stmt.args) == 4 - arr = stmt.args[3] - mark_use(arr, idx) - - # elseif is_known_call(stmt, setindex!, compact) && length(stmt.args) == 4 - # # handle similarly to arrayset - # val = stmt.args[3] - # mark_escape(val) - - # arr = stmt.args[2] - # mark_use(arr, idx) - - elseif is_known_call(stmt, (===), compact) && length(stmt.args) == 3 - arr1 = stmt.args[2] - arr2 = stmt.args[3] - - mark_use(arr1, idx) - mark_use(arr2, idx) - - # these foreigncalls have similar structure and don't escape our array, so handle them all at once - elseif is_known_fcall(stmt, (:jl_array_ptr, :jl_array_copy)) && length(stmt.args) == 6 - arr = stmt.args[6] - mark_use(arr, idx) - - elseif is_known_call(stmt, arraysize, compact) && isa(stmt.args[2], SSAValue) - arr = stmt.args[2] - mark_use(arr, idx) - + # elseif is_array_allocation(stmt) + # push!(relevant, idx) elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue) - # mark these for potential replacement with mutating_arrayfreeze push!(revisit, idx) - - else - # Assume everything else escapes - for ur in userefs(stmt) - mark_escape(ur[]) - end end end @@ -1397,13 +1301,11 @@ function memory_opt!(ir::IRCode) domtree = construct_domtree(ir.cfg.blocks) for idx in revisit - # Make sure that the value we reference didn't escape stmt = ir.stmts[idx][:inst]::Expr - #println("test, stmt : ", stmt) - id = (stmt.args[2]::SSAValue).id - (id in relevant) || continue - - #println("Revisiting ", stmt) + id = stmt.args[2].id + # if our escape analysis has determined that this array doesn't escape, we can potentially eliminate an allocation + # @eval Main (ir = $ir; rev = $revisit; esc_state = $escape_state) + has_no_escape(escape_state.ssavalues[id]) || continue # We're ok to steal the memory if we don't dominate any uses ok = true @@ -1416,6 +1318,7 @@ function memory_opt!(ir::IRCode) end end ok || continue + println("saved an allocation here :", stmt) stmt.args[1] = Core.mutating_arrayfreeze end @@ -1426,14 +1329,6 @@ function memory_opt!(ir::IRCode) # #println(stmt.args) # arr = stmt.args[2] # id = isa(arr, SSAValue) ? arr.id : arr.n # SSAValue or Core.Argument - - # if (id in relevant) # didn't escape elsewhere, so make a copy to keep it un-escaped - # #println("didn't escape maybecopy") - # stmt.args[1] = Main.Base.copy - # else # already escaped, so save the cost of copying and just pass the actual object - # #println("escaped maybecopy") - # ir.stmts[idx][:inst] = arr - # end # end return ir diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index d4dc0d85da3a5..0c49dd7710f38 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,4 +1,3 @@ -using Base.Experimental: ImmutableArray using Test function test_allocate1() From 49f93377c7677c7d602a03462be9897b75d65726 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 21 Dec 2021 14:55:02 +0900 Subject: [PATCH 05/41] fixup definitions of array primitives --- base/abstractarray.jl | 5 +---- base/array.jl | 28 +--------------------------- base/boot.jl | 11 +++++++++-- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 2faa8d27e5861..687bdeeb35dae 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1078,10 +1078,7 @@ function copy(a::AbstractArray) @_propagate_inbounds_meta copymutable(a) end - -function copy(a::Core.ImmutableArray) - a -end +copy(a::Core.ImmutableArray) = a function copyto!(B::AbstractVecOrMat{R}, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, A::AbstractVecOrMat{S}, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) where {R,S} diff --git a/base/array.jl b/base/array.jl index a259172d28c73..e8c3b59a85b4b 100644 --- a/base/array.jl +++ b/base/array.jl @@ -176,12 +176,6 @@ function vect(X...) return copyto!(Vector{T}(undef, length(X)), X) end -# Freeze and thaw constructors -ImmutableArray(a::Array) = Core.arrayfreeze(a) -Array(a::ImmutableArray) = Core.arraythaw(a) - -ImmutableArray(a::AbstractArray{T,N}) where {T,N} = ImmutableArray{T,N}(a) - # Size functions for arrays, both mutable and immutable size(a::IMArray, d::Integer) = arraysize(a, convert(Int, d)) size(a::IMVector) = (arraysize(a,1),) @@ -259,20 +253,6 @@ function isassigned(a::IMArray, i::Int...) ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 end -# function isassigned(a::Array, i::Int...) -# @_inline_meta -# ii = (_sub2ind(size(a), i...) % UInt) - 1 -# @boundscheck ii < length(a) % UInt || return false -# ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 -# end - -# function isassigned(a::ImmutableArray, i::Int...) -# @_inline_meta -# ii = (_sub2ind(size(a), i...) % UInt) - 1 -# @boundscheck ii < length(a) % UInt || return false -# ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 -# end - ## copy ## """ @@ -428,9 +408,6 @@ similar(a::Array{T}, m::Int) where {T} = Vector{T}(undef, m) similar(a::Array, T::Type, dims::Dims{N}) where {N} = Array{T,N}(undef, dims) similar(a::Array{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) -ImmutableArray{T}(undef::UndefInitializer, m::Int) where T = ImmutableArray(Array{T}(undef, m)) -ImmutableArray{T}(undef::UndefInitializer, dims::Dims) where T = ImmutableArray(Array{T}(undef, dims)) - """ maybecopy(x) @@ -674,7 +651,7 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) ## Conversions ## convert(::Type{Union{}}, a::AbstractArray) = throw(MethodError(convert, (Union{}, a))) -convert(T::Union{Type{<:Array},Type{<:Core.ImmutableArray}}, a::AbstractArray) = a isa T ? a : T(a) +convert(T::Type{<:IMArray}, a::AbstractArray) = a isa T ? a : T(a) promote_rule(a::Type{Array{T,n}}, b::Type{Array{S,n}}) where {T,n,S} = el_same(promote_type(T,S), a, b) @@ -987,9 +964,6 @@ function getindex end @eval getindex(A::ImmutableArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) @eval getindex(A::ImmutableArray, i1::Int, i2::Int, I::Int...) = (@inline; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) -# @eval getindex(A::IMArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) -# @eval getindex(A::IMArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) - # Faster contiguous indexing using copyto! for AbstractUnitRange and Colon function getindex(A::Array, I::AbstractUnitRange{<:Integer}) @inline diff --git a/base/boot.jl b/base/boot.jl index e520363a840bc..433eacc219ffb 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -475,11 +475,10 @@ Array{T,N}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = ccall(:jl_new_arr Array{T}(::UndefInitializer, m::Int) where {T} = Array{T,1}(undef, m) Array{T}(::UndefInitializer, m::Int, n::Int) where {T} = Array{T,2}(undef, m, n) Array{T}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = Array{T,3}(undef, m, n, o) -Array{T}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = Array{T,N}(undef, d) +Array{T}(::UndefInitializer, d::NTuple{N,Int}#=::Dims=#) where {T,N} = Array{T,N}(undef, d) # empty vector constructor Array{T,1}() where {T} = Array{T,1}(undef, 0) - (Array{T,N} where T)(x::AbstractArray{S,N}) where {S,N} = Array{S,N}(x) Array(A::AbstractArray{T,N}) where {T,N} = Array{T,N}(A) @@ -487,6 +486,14 @@ Array{T}(A::AbstractArray{S,N}) where {T,N,S} = Array{T,N}(A) AbstractArray{T}(A::AbstractArray{S,N}) where {T,S,N} = AbstractArray{T,N}(A) +# freeze and thaw constructors +ImmutableArray(a::Array) = arrayfreeze(a) +ImmutableArray(a::AbstractArray{T,N}) where {T,N} = ImmutableArray{T,N}(a) +Array(a::ImmutableArray) = arraythaw(a) +# undef initializers +ImmutableArray{T,N}(::UndefInitializer, args...) where {T,N} = ImmutableArray(Array{T,N}(undef, args...)) +ImmutableArray{T}(::UndefInitializer, args...) where {T} = ImmutableArray(Array{T}(undef, args...)) + # primitive Symbol constructors eval(Core, :(function Symbol(s::String) $(Expr(:meta, :pure)) From b02e7c054b5b7df6848b8075ea1c8a3520e6d666 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 21 Dec 2021 14:18:34 +0900 Subject: [PATCH 06/41] fixup `@noinline` annotations within `BoundsError` --- base/boot.jl | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index 433eacc219ffb..8a6aa7bdc035e 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -278,13 +278,10 @@ struct BoundsError <: Exception # Eventually, we want to figure out if the copy is needed to save the performance of copying # (i.e., if a escapes elsewhere, don't bother to make a copy) - BoundsError(@nospecialize(a)) = a isa Array ? - (@noinline; new(Core.maybecopy(a))) : - (@noinline; new(a)) - - BoundsError(@nospecialize(a), i) = a isa Array ? - (@noinline; new(Core.maybecopy(a), i)) : - (@noinline; new(a, i)) + BoundsError(@nospecialize(a)) = (@noinline; + a isa Array ? new(Core.maybecopy(a)) : new(a)) + BoundsError(@nospecialize(a), i) = (@noinline; + a isa Array ? new(Core.maybecopy(a), i) : new(a, i)) end struct DivideError <: Exception end struct OutOfMemoryError <: Exception end From 2a52e3d33e64c63ee0a7ae6f526c0ccb275aef2d Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 21 Dec 2021 15:04:59 +0900 Subject: [PATCH 07/41] disable EA global caching to succeed in sysimg building --- base/compiler/bootstrap.jl | 2 +- base/compiler/optimize.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/compiler/bootstrap.jl b/base/compiler/bootstrap.jl index 2517b181d2804..8e4743a28f149 100644 --- a/base/compiler/bootstrap.jl +++ b/base/compiler/bootstrap.jl @@ -14,7 +14,7 @@ let fs = Any[ # we first create caches for the optimizer, because they contain many loop constructions # and they're better to not run in interpreter even during bootstrapping - run_passes, + find_escapes, run_passes, # then we create caches for inference entries typeinf_ext, typeinf, typeinf_edge, ] diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 78d318fba2fab..a2e132cd90a60 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -538,13 +538,13 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) @timeit "SROA" ir = sroa_pass!(ir) @timeit "ADCE" ir = adce_pass!(ir) @timeit "type lift" ir = type_lift_pass!(ir) - # @timeit "compact 3" ir = compact!(ir) nargs = let def = sv.linfo.def isa(def, Method) ? Int(def.nargs) : 0 end esc_state = find_escapes(ir, nargs) - setindex!(GLOBAL_ESCAPE_CACHE, esc_state, sv.linfo) - ir = memory_opt!(ir, esc_state) + # setindex!(GLOBAL_ESCAPE_CACHE, esc_state, sv.linfo) + @timeit "memory opt" ir = memory_opt!(ir, esc_state) + # @timeit "compact 3" ir = compact!(ir) if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) end From bd0bdb713233cff2c8851ffac4e6e83151257102 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 21 Dec 2021 16:39:12 +0900 Subject: [PATCH 08/41] define EA in a separate module --- base/compiler/EscapeAnalysis.jl | 544 ++++++++++++++++++++++++++++++++ base/compiler/compiler.jl | 2 + base/compiler/optimize.jl | 412 ------------------------ base/compiler/ssair/passes.jl | 21 +- 4 files changed, 561 insertions(+), 418 deletions(-) create mode 100644 base/compiler/EscapeAnalysis.jl diff --git a/base/compiler/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis.jl new file mode 100644 index 0000000000000..ebee3ff53bc00 --- /dev/null +++ b/base/compiler/EscapeAnalysis.jl @@ -0,0 +1,544 @@ +baremodule EscapeAnalysis + +export + find_escapes, + has_not_analyzed, + has_no_escape, + has_return_escape, + has_thrown_escape, + has_all_escape, + can_elide_finalizer + +# analysis +# ======== + +const _TOP_MOD = ccall(:jl_base_relative_to, Any, (Any,), EscapeAnalysis)::Module + +# imports +import ._TOP_MOD: == +# usings +import Core: + MethodInstance, + Const, + Argument, + SSAValue, + PiNode, + PhiNode, + UpsilonNode, + PhiCNode, + ReturnNode, + GotoNode, + GotoIfNot, + SimpleVector +import ._TOP_MOD: # Base definitions + @eval, @assert, @nospecialize, @__MODULE__, Vector, BitSet, IdDict, + !, !==, ≠, +, -, ≤, &, |, include, error, missing, println, + ∪, ⊆, ∩, :, length, get, first, last, in, isempty, isassigned, push!, empty! +import Core.Compiler: # Core.Compiler specific definitions + IRCode, IR_FLAG_EFFECT_FREE, + isbitstype, isexpr, is_meta_expr_head, widenconst, argextype, singleton_type + +""" + x::EscapeLattice + +A lattice for escape information, which holds the following properties: +- `x.Analyzed::Bool`: not formally part of the lattice, indicates `x` has not been analyzed at all +- `x.ReturnEscape::Bool`: indicates `x` may escape to the caller via return (possibly as a field), + where `x.ReturnEscape && 0 ∈ x.EscapeSites` has the special meaning that it's visible to + the caller simply because it's passed as call argument +- `x.ThrownEscape::Bool`: indicates `x` may escape to somewhere through an exception (possibly as a field) +- `x.EscapeSites::BitSet`: records program counters (SSA numbers) where `x` can escape +- `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through `setfield!` on argument(s) + * `-1` : no escape + * `0` : unknown or multiple + * `n` : through argument N + +These attributes can be combined to create a partial lattice that has a finite height, given +that input program has a finite number of statements, which is assured by Julia's semantics. + +There are utility constructors to create common `EscapeLattice`s, e.g., +- `NoEscape()`: the bottom element of this lattice, meaning it won't escape to anywhere +- `AllEscape()`: the topmost element of this lattice, meaning it will escape to everywhere + +The escape analysis will transition these elements from the bottom to the top, +in the same direction as Julia's native type inference routine. +An abstract state will be initialized with the bottom(-like) elements: +- the call arguments are initialized as `ArgumentReturnEscape()`, because they're visible from a caller immediately +- the other states are initialized as `NotAnalyzed()`, which is a special lattice element that + is slightly lower than `NoEscape`, but at the same time doesn't represent any meaning + other than it's not analyzed yet (thus it's not formally part of the lattice). +""" +struct EscapeLattice + Analyzed::Bool + ReturnEscape::Bool + ThrownEscape::Bool + EscapeSites::BitSet + # TODO: ArgEscape::Int +end + +# precomputed default values in order to eliminate computations at each callsite +const EMPTY_ESCAPE_SITES = BitSet() +const ARGUMENT_ESCAPE_SITES = BitSet(0) + +# the constructors +NotAnalyzed() = EscapeLattice(false, false, false, EMPTY_ESCAPE_SITES) # not formally part of the lattice +NoEscape() = EscapeLattice(true, false, false, EMPTY_ESCAPE_SITES) +ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc)) +ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc)) +ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES) +let + all_escape_sites = BitSet(0:100_000) + global AllEscape() = EscapeLattice(true, true, true, all_escape_sites) + # used for `show` + global AllReturnEscape() = EscapeLattice(true, true, false, all_escape_sites) + global AllThrownEscape() = EscapeLattice(true, false, true, all_escape_sites) +end + +# Convenience names for some ⊑ queries +export + has_not_analyzed, + has_no_escape, + has_return_escape, + has_thrown_escape, + has_all_escape, + can_elide_finalizer +has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() +has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() +has_return_escape(x::EscapeLattice) = x.ReturnEscape +has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites +has_thrown_escape(x::EscapeLattice) = x.ThrownEscape +has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites +has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x + +""" + can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool + +Queries the validity of the finalizer elision optimization at the `return` site of statement `pc`, +which inserts `finalize` call when the lifetime of interested object ends. +Note that we don't need to take `x.ThrownEscape` into account because it would have never +been thrown when the program execution reaches the `return` site. +""" +can_elide_finalizer(x::EscapeLattice, pc::Int) = + !(has_return_escape(x, 0) || has_return_escape(x, pc)) + +# we need to make sure this `==` operator corresponds to lattice equality rather than object equality, +# otherwise `propagate_changes` can't detect the convergence +x::EscapeLattice == y::EscapeLattice = begin + return x.Analyzed === y.Analyzed && + x.ReturnEscape === y.ReturnEscape && + x.ThrownEscape === y.ThrownEscape && + x.EscapeSites == y.EscapeSites && + true +end + +x::EscapeLattice ⊑ y::EscapeLattice = begin + if x.Analyzed ≤ y.Analyzed && + x.ReturnEscape ≤ y.ReturnEscape && + x.ThrownEscape ≤ y.ThrownEscape && + x.EscapeSites ⊆ y.EscapeSites && + true + return true + end + return false +end +x::EscapeLattice ⊏ y::EscapeLattice = x ⊑ y && !(y ⊑ x) +x::EscapeLattice ⋤ y::EscapeLattice = !(y ⊑ x) + +x::EscapeLattice ⊔ y::EscapeLattice = begin + return EscapeLattice( + x.Analyzed | y.Analyzed, + x.ReturnEscape | y.ReturnEscape, + x.ThrownEscape | y.ThrownEscape, + x.EscapeSites ∪ y.EscapeSites, + ) +end + +x::EscapeLattice ⊓ y::EscapeLattice = begin + return EscapeLattice( + x.Analyzed & y.Analyzed, + x.ReturnEscape & y.ReturnEscape, + x.ThrownEscape & y.ThrownEscape, + x.EscapeSites ∩ y.EscapeSites, + ) +end + +# TODO setup a more effient struct for cache +# which can discard escape information on SSS values and arguments that don't join dispatch signature + +""" + state::EscapeState + +Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`: +- `state.arguments::Vector{EscapeLattice}`: escape information about "arguments" – note that + "argument" can include both call arguments and slots appearing in analysis frame +- `ssavalues::Vector{EscapeLattice}`: escape information about each SSA value +""" +struct EscapeState + arguments::Vector{EscapeLattice} + ssavalues::Vector{EscapeLattice} +end +function EscapeState(nslots::Int, nargs::Int, nstmts::Int) + arguments = EscapeLattice[ + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] + ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] + return EscapeState(arguments, ssavalues) +end + +# we preserve `IRCode` as well just for debugging purpose +const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Tuple{EscapeState,IRCode}}() +__clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) + +const Change = Pair{Union{Argument,SSAValue},EscapeLattice} +const Changes = Vector{Change} + +""" + find_escapes(ir::IRCode, nargs::Int) -> EscapeState + +Escape analysis implementation is based on the data-flow algorithm described in the paper [^MM02]. +The analysis works on the lattice of [`EscapeLattice`](@ref) and transitions lattice elements +from the bottom to the top in a _backward_ way, i.e. data flows from usage cites to definitions, +until every lattice gets converged to a fixed point by maintaining a (conceptual) working set +that contains program counters corresponding to remaining SSA statements to be analyzed. +The analysis only manages a single global state that tracks `EscapeLattice` of each argument +and SSA statement, but also note that some flow-sensitivity is encoded as program counters +recorded in the `EscapeSites` property of each each lattice element. + +[^MM02]: _A Graph-Free approach to Data-Flow Analysis_. + Markas Mohnen, 2002, April. + . +""" +function find_escapes(ir::IRCode, nargs::Int) + (; stmts, sptypes, argtypes) = ir + nstmts = length(stmts) + + # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties + state = EscapeState(length(argtypes), nargs, nstmts) + changes = Changes() # stashes changes that happen at current statement + + while true + local anyupdate = false + + for pc in nstmts:-1:1 + stmt = stmts.inst[pc] + + # we escape statements with the `ThrownEscape` property using the effect-freeness + # information computed by the inliner + is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 + + # collect escape information + if isa(stmt, Expr) + head = stmt.head + if head === :call + has_changes = escape_call!(ir, pc, stmt.args, state, changes) + if !is_effect_free + for x in stmt.args + add_change!(x, ir, ThrownEscape(pc), changes) + end + else + has_changes || continue + end + elseif head === :invoke + escape_invoke!(ir, pc, stmt.args, state, changes) + elseif head === :new || head === :splatnew + escape_new!(ir, pc, stmt.args, state, changes) + elseif head === :(=) + lhs, rhs = stmt.args + if isa(lhs, GlobalRef) # global store + add_change!(rhs, ir, AllEscape(), changes) + end + elseif head === :foreigncall + escape_foreigncall!(ir, pc, stmt.args, state, changes) + elseif head === :throw_undef_if_not # XXX when is this expression inserted ? + add_change!(stmt.args[1], ir, ThrownEscape(pc), changes) + elseif is_meta_expr_head(head) + # meta expressions doesn't account for any usages + continue + elseif head === :static_parameter + # :static_parameter refers any of static parameters, but since they exist + # statically, we're really not interested in their escapes + continue + elseif head === :copyast + # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values + continue + elseif head === :undefcheck + # undefcheck is temporarily inserted by compiler + # it will be processd be later pass so it won't change any of escape states + continue + elseif head === :the_exception + # we don't propagate escape information on exceptions via this expression, but rather + # use a dedicated lattice property `ThrownEscape` + continue + elseif head === :isdefined + # just returns `Bool`, nothing accounts for any usages + continue + elseif head === :enter || head === :leave || head === :pop_exception + # these exception frame managements doesn't account for any usages + # we can just ignore escape information from + continue + elseif head === :gc_preserve_begin || head === :gc_preserve_end + # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information + # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere + continue + else + for x in stmt.args + add_change!(x, ir, AllEscape(), changes) + end + end + elseif isa(stmt, GlobalRef) # global load + add_change!(SSAValue(pc), ir, AllEscape(), changes) + elseif isa(stmt, PiNode) + if isdefined(stmt, :val) + info = state.ssavalues[pc] + add_change!(stmt.val, ir, info, changes) + end + elseif isa(stmt, PhiNode) + escape_backedges!(ir, pc, stmt.values, state, changes) + elseif isa(stmt, PhiCNode) + escape_backedges!(ir, pc, stmt.values, state, changes) + elseif isa(stmt, UpsilonNode) + if isdefined(stmt, :val) + info = state.ssavalues[pc] + add_change!(stmt.val, ir, info, changes) + end + elseif isa(stmt, ReturnNode) + if isdefined(stmt, :val) + add_change!(stmt.val, ir, ReturnEscape(pc), changes) + end + elseif isa(stmt, SSAValue) + # NOTE after SROA, we may see SSA value as statement + info = state.ssavalues[pc] + add_change!(stmt, ir, info, changes) + else + @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me + continue + end + + isempty(changes) && continue + + anyupdate |= propagate_changes!(state, changes) + + empty!(changes) + end + + anyupdate || break + end + + return state +end + +# propagate changes, and check convergence +function propagate_changes!(state::EscapeState, changes::Changes) + local anychanged = false + + for (x, info) in changes + if isa(x, Argument) + old = state.arguments[x.n] + new = old ⊔ info + if old ≠ new + state.arguments[x.n] = new + anychanged |= true + end + else + x = x::SSAValue + old = state.ssavalues[x.id] + new = old ⊔ info + if old ≠ new + state.ssavalues[x.id] = new + anychanged |= true + end + end + end + + return anychanged +end + +function add_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) + if isa(x, Argument) || isa(x, SSAValue) + if !isbitstype(widenconst(argextype(x, ir, ir.sptypes, ir.argtypes))) + push!(changes, Change(x, info)) + end + end +end + +function escape_backedges!(ir::IRCode, pc::Int, backedges::Vector{Any}, + state::EscapeState, changes::Changes) + info = state.ssavalues[pc] + for i in 1:length(backedges) + if isassigned(backedges, i) + add_change!(backedges[i], ir, info, changes) + end + end +end + +function escape_call!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) + f = singleton_type(ft) + if isa(f, Core.IntrinsicFunction) + return false # COMBAK we may break soundness here, e.g. `pointerref` + end + result = escape_builtin!(f, ir, pc, args, state, changes) + if result === false + return false # nothing to propagate + elseif result === missing + # if this call hasn't been handled by any of pre-defined handlers, + # we escape this call conservatively + for i in 2:length(args) + add_change!(args[i], ir, AllEscape(), changes) + end + return true + else + return true + end +end + +function escape_invoke!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + linfo = first(args)::MethodInstance + cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) + args = args[2:end] + if cache === nothing + for x in args + add_change!(x, ir, AllEscape(), changes) + end + else + (linfostate, _ #=ir::IRCode=#) = cache + retinfo = state.ssavalues[pc] # escape information imposed on the call statement + method = linfo.def::Method + nargs = Int(method.nargs) + for i in 1:length(args) + arg = args[i] + if i ≤ nargs + arginfo = linfostate.arguments[i] + else # handle isva signature: COMBAK will this invalid once we encode alias information ? + arginfo = linfostate.arguments[nargs] + end + if isempty(arginfo.ReturnEscape) + @eval Main (ir = $ir; linfo = $linfo) + error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") + end + info = from_interprocedural(arginfo, retinfo, pc) + add_change!(arg, ir, info, changes) + end + end +end + +# reinterpret the escape information imposed on the callee argument (`arginfo`) in the +# context of the caller frame using the escape information imposed on the return value (`retinfo`) +function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc::Int) + @assert arginfo.ReturnEscape + if arginfo.ThrownEscape + EscapeSites = BitSet(pc) + else + EscapeSites = EMPTY_ESCAPE_SITES + end + newarginfo = EscapeLattice(true, false, arginfo.ThrownEscape, EscapeSites) + if arginfo.EscapeSites === ARGUMENT_ESCAPE_SITES + # if this is simply passed as the call argument, we can discard the `ReturnEscape` + # information and just propagate the other escape information + return newarginfo + else + # if this can be returned, we have to merge its escape information with + # that of the current statement + return newarginfo ⊔ retinfo + end +end + +function escape_new!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not + end + + # propagate the escape information of this object to all its fields as well + # since they can be accessed through the object + for i in 2:length(args) + add_change!(args[i], ir, info, changes) + end +end + +# escape every argument `(args[6:length(args[3])])` and the name `args[1]` +# TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls +function escape_foreigncall!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + foreigncall_nargs = length((args[3])::SimpleVector) + name = args[1] + # if normalize(name) === :jl_gc_add_finalizer_th + # # add `FinalizerEscape` ? + # end + add_change!(name, ir, ThrownEscape(pc), changes) + for i in 6:5+foreigncall_nargs + add_change!(args[i], ir, ThrownEscape(pc), changes) + end +end + +# NOTE error cases will be handled in `find_escapes` anyway, so we don't need to take care of them below +# TODO implement more builtins, make them more accurate +# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? + +escape_builtin!(@nospecialize(f), _...) = return missing + +# safe builtins +escape_builtin!(::typeof(isa), _...) = return false +escape_builtin!(::typeof(typeof), _...) = return false +escape_builtin!(::typeof(Core.sizeof), _...) = return false +escape_builtin!(::typeof(===), _...) = return false +# not really safe, but `ThrownEscape` will be imposed later +escape_builtin!(::typeof(isdefined), _...) = return false +escape_builtin!(::typeof(throw), _...) = return false + +function escape_builtin!(::typeof(Core.ifelse), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) == 4 || return + f, cond, th, el = args + info = state.ssavalues[pc] + condt = argextype(cond, ir, ir.sptypes, ir.argtypes) + if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) + if cond + add_change!(th, ir, info, changes) + else + add_change!(el, ir, info, changes) + end + else + add_change!(th, ir, info, changes) + add_change!(el, ir, info, changes) + end +end + +function escape_builtin!(::typeof(typeassert), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) == 3 || return + f, obj, typ = args + info = state.ssavalues[pc] + add_change!(obj, ir, info, changes) +end + +function escape_builtin!(::typeof(tuple), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + end + for i in 2:length(args) + add_change!(args[i], ir, info, changes) + end +end + +# TODO don't propagate escape information to the 1st argument, but propagate information to aliased field +function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + # only propagate info when the field itself is non-bitstype + isbitstype(widenconst(ir.stmts.type[pc])) && return true + info = state.ssavalues[pc] + if info == NotAnalyzed() + info = NoEscape() + end + for i in 2:length(args) + add_change!(args[i], ir, info, changes) + end +end + +# NOTE define fancy package utilities when developing EA as an external package +if !(_TOP_MOD === Core.Compiler) + include(@__MODULE__, "utils.jl") +end + +end # baremodule EscapeAnalysis diff --git a/base/compiler/compiler.jl b/base/compiler/compiler.jl index c265512afcbf6..30054f51adeb6 100644 --- a/base/compiler/compiler.jl +++ b/base/compiler/compiler.jl @@ -130,6 +130,8 @@ include("compiler/stmtinfo.jl") include("compiler/abstractinterpretation.jl") include("compiler/typeinfer.jl") include("compiler/optimize.jl") # TODO: break this up further + extract utilities +include("compiler/EscapeAnalysis.jl") +using .EscapeAnalysis include("compiler/bootstrap.jl") ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index a2e132cd90a60..9f8f9f8ae6c56 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -121,93 +121,6 @@ function ir_to_codeinf!(opt::OptimizationState) return src end -################## -# EscapeAnalysis # -################## - -struct EscapeLattice - Analyzed::Bool - ReturnEscape - ThrownEscape::Bool - GlobalEscape::Bool - # TODO: ArgEscape::Int -end - -function (==)(x::EscapeLattice, y::EscapeLattice) - return x.Analyzed === y.Analyzed && - x.ReturnEscape == y.ReturnEscape && - x.ThrownEscape === y.ThrownEscape && - x.GlobalEscape === y.GlobalEscape -end - -const NO_RETURN = BitSet() -const ARGUMENT_RETURN = BitSet(0) -NotAnalyzed() = EscapeLattice(false, NO_RETURN, false, false) # not formally part of the lattice -NoEscape() = EscapeLattice(true, NO_RETURN, false, false) -ReturnEscape(pcs::BitSet) = EscapeLattice(true, pcs, false, false) -ReturnEscape(pc::Int) = ReturnEscape(BitSet(pc)) -ArgumentReturnEscape() = ReturnEscape(ARGUMENT_RETURN) -ThrownEscape() = EscapeLattice(true, NO_RETURN, true, false) -GlobalEscape() = EscapeLattice(true, NO_RETURN, false, true) -let - all_return = BitSet(0:100_000) - global AllReturnEscape() = ReturnEscape(all_return) # used for `show` - global AllEscape() = EscapeLattice(true, all_return, true, true) -end - -function ⊑(x::EscapeLattice, y::EscapeLattice) - if x.Analyzed ≤ y.Analyzed && - x.ReturnEscape ⊆ y.ReturnEscape && - x.ThrownEscape ≤ y.ThrownEscape && - x.GlobalEscape ≤ y.GlobalEscape - return true - end - return false -end - -⋤(x::EscapeLattice, y::EscapeLattice) = ⊑(x, y) && !⊑(y, x) - -function ⊔(x::EscapeLattice, y::EscapeLattice) - return EscapeLattice( - x.Analyzed | y.Analyzed, - x.ReturnEscape ∪ y.ReturnEscape, - x.ThrownEscape | y.ThrownEscape, - x.GlobalEscape | y.GlobalEscape, - ) -end - -function ⊓(x::EscapeLattice, y::EscapeLattice) - return EscapeLattice( - x.Analyzed & y.Analyzed, - x.ReturnEscape ∩ y.ReturnEscape, - x.ThrownEscape & y.ThrownEscape, - x.GlobalEscape & y.GlobalEscape, - ) -end - -has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() -has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() -has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) -has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape -has_thrown_escape(x::EscapeLattice) = x.ThrownEscape -has_global_escape(x::EscapeLattice) = x.GlobalEscape -has_all_escape(x::EscapeLattice) = AllEscape() == x - -const Change = Pair{Union{Argument,SSAValue},EscapeLattice} -const Changes = Vector{Change} - -struct EscapeState - arguments::Vector{EscapeLattice} - ssavalues::Vector{EscapeLattice} -end - -function EscapeState(nslots::Int, nargs::Int, nstmts::Int) - arguments = EscapeLattice[ - 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] - ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] - return EscapeState(arguments, ssavalues) -end - ############# # constants # ############# @@ -248,9 +161,6 @@ const _PURE_OR_ERROR_BUILTINS = [ const TOP_TUPLE = GlobalRef(Core, :tuple) -const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance, EscapeState}() -__clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) - ######### # logic # ######### @@ -797,17 +707,6 @@ function statement_costs!(cost::Vector{Int}, body::Vector{Any}, src::Union{CodeI return maxcost end -function is_known_fcall(stmt::Expr, funcs) - isexpr(stmt, :foreigncall) || return false - s = stmt.args[1] - isa(s, QuoteNode) && (s = s.value) - isa(s, Symbol) || return false - for func in funcs - s === func && return true - end - return false -end - function renumber_ir_elements!(body::Vector{Any}, changemap::Vector{Int}) return renumber_ir_elements!(body, changemap, changemap) end @@ -889,314 +788,3 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab end end end - -function propagate_changes!(state::EscapeState, changes::Changes) - local anychanged = false - - for (x, info) in changes - if isa(x, Argument) - old = state.arguments[x.n] - new = old ⊔ info - if old ≠ new - state.arguments[x.n] = new - anychanged |= true - end - else - x = x::SSAValue - old = state.ssavalues[x.id] - new = old ⊔ info - if old ≠ new - state.ssavalues[x.id] = new - anychanged |= true - end - end - end - - return anychanged -end - -# function normalize(@nospecialize(x)) -# if isa(x, QuoteNode) -# return x.value -# else -# return x -# end -# end - -function add_changes!(args::Vector{Any}, ir::IRCode, info::EscapeLattice, changes::Changes) - for x in args - add_change!(x, ir, info, changes) - end -end - -function add_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) - if isa(x, Argument) || isa(x, SSAValue) - if !isbitstype(widenconst(argextype(x, ir, ir.sptypes, ir.argtypes))) - push!(changes, Change(x, info)) - end - end -end - -function escape_invoke!(args::Vector{Any}, pc::Int, - state::EscapeState, ir::IRCode, changes::Changes) - linfo = first(args)::MethodInstance - cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) - args = args[2:end] - if cache === nothing - add_changes!(args, ir, AllEscape(), changes) - else - linfostate = cache - retinfo = state.ssavalues[pc] # escape information imposed on the call statement - method = linfo.def::Method - nargs = Int(method.nargs) - for i in 1:length(args) - arg = args[i] - if i ≤ nargs - arginfo = linfostate.arguments[i] - else # handle isva signature: COMBAK will this invalid once we encode alias information ? - arginfo = linfostate.arguments[nargs] - end - if isempty(arginfo.ReturnEscape) - @eval Main (ir = $ir; linfo = $linfo) - error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") - end - info = from_interprocedural(arginfo, retinfo) - add_change!(arg, ir, info, changes) - end - end -end - -# reinterpret the escape information imposed on the callee argument (`arginfo`) in the -# context of the caller frame using the escape information imposed on the return value (`retinfo`) -function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice) - ar = arginfo.ReturnEscape - newarginfo = EscapeLattice(true, NO_RETURN, arginfo.ThrownEscape, arginfo.GlobalEscape) - if ar == ARGUMENT_RETURN - # if this is simply passed as the call argument, we can discard the `ReturnEscape` - # information and just propagate the other escape information - return newarginfo - else - # if this can be a return value, we have to merge it with the escape information - return newarginfo ⊔ retinfo - end -end - -function escape_call!(args::Vector{Any}, pc::Int, - state::EscapeState, ir::IRCode, changes::Changes) - ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) - f = singleton_type(ft) - if isa(f, Core.IntrinsicFunction) - return false # COMBAK we may break soundness here, e.g. `pointerref` - end - ishandled = escape_builtin!(f, args, pc, state, ir, changes)::Union{Nothing,Bool} - ishandled === nothing && return false # nothing to propagate - if !ishandled - # if this call hasn't been handled by any of pre-defined handlers, - # we escape this call conservatively - add_changes!(args[2:end], ir, AllEscape(), changes) - end - return true -end - -# TODO implement more builtins, make them more accurate -# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? - -escape_builtin!(@nospecialize(f), _...) = return false - -escape_builtin!(::typeof(isa), _...) = return nothing -escape_builtin!(::typeof(typeof), _...) = return nothing -escape_builtin!(::typeof(Core.sizeof), _...) = return nothing -escape_builtin!(::typeof(===), _...) = return nothing - -function escape_builtin!(::typeof(Core.ifelse), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) - length(args) == 4 || return false - f, cond, th, el = args - info = state.ssavalues[pc] - condt = argextype(cond, ir, ir.sptypes, ir.argtypes) - if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) - if cond - add_change!(th, ir, info, changes) - else - add_change!(el, ir, info, changes) - end - else - add_change!(th, ir, info, changes) - add_change!(el, ir, info, changes) - end - return true -end - -function escape_builtin!(::typeof(typeassert), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) - length(args) == 3 || return false - f, obj, typ = args - info = state.ssavalues[pc] - add_change!(obj, ir, info, changes) - return true -end - -function escape_builtin!(::typeof(tuple), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - end - add_changes!(args[2:end], ir, info, changes) - return true -end - -# TODO don't propagate escape information to the 1st argument, but propagate information to aliased field -function escape_builtin!(::typeof(getfield), args::Vector{Any}, pc::Int, state::EscapeState, ir::IRCode, changes::Changes) - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - end - # only propagate info when the field itself is non-bitstype - if !isbitstype(widenconst(ir.stmts.type[pc])) - add_changes!(args[2:end], ir, info, changes) - end - return true -end - -function find_escapes(ir::IRCode, nargs::Int) - (; stmts, sptypes, argtypes) = ir - nstmts = length(stmts) - - # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties - state = EscapeState(length(ir.argtypes), nargs, nstmts) - changes = Changes() # stashes changes that happen at current statement - - while true - local anyupdate = false - - for pc in nstmts:-1:1 - stmt = stmts.inst[pc] - - # we escape statements with the `ThrownEscape` property using the effect-freeness - # information computed by the inliner - is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 - - # collect escape information - if isa(stmt, Expr) - head = stmt.head - if head === :call - has_changes = escape_call!(stmt.args, pc, state, ir, changes) - if !is_effect_free - add_changes!(stmt.args, ir, ThrownEscape(), changes) - else - has_changes || continue - end - elseif head === :invoke - escape_invoke!(stmt.args, pc, state, ir, changes) - elseif head === :new - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not - end - add_changes!(stmt.args[2:end], ir, info, changes) - elseif head === :splatnew - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not - end - # splatnew passes field values using a single tuple (args[2]) - add_change!(stmt.args[2], ir, info, changes) - elseif head === :(=) - lhs, rhs = stmt.args - if isa(lhs, GlobalRef) # global store - add_change!(rhs, ir, GlobalEscape(), changes) - end - elseif head === :foreigncall - # for foreigncall we simply escape every argument (args[6:length(args[3])]) - # and its name (args[1]) - # TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls - foreigncall_nargs = length((stmt.args[3])::SimpleVector) - name = stmt.args[1] - # if normalize(name) === :jl_gc_add_finalizer_th - # continue # XXX assume this finalizer call is valid for finalizer elision - # end - add_change!(name, ir, ThrownEscape(), changes) - add_changes!(stmt.args[6:5+foreigncall_nargs], ir, ThrownEscape(), changes) - elseif head === :throw_undef_if_not # XXX when is this expression inserted ? - add_change!(stmt.args[1], ir, ThrownEscape(), changes) - elseif is_meta_expr_head(head) - # meta expressions doesn't account for any usages - continue - elseif head === :static_parameter - # :static_parameter refers any of static parameters, but since they exist - # statically, we're really not interested in their escapes - continue - elseif head === :copyast - # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values - continue - elseif head === :undefcheck - # undefcheck is temporarily inserted by compiler - # it will be processd be later pass so it won't change any of escape states - continue - elseif head === :the_exception - # we don't propagate escape information on exceptions via this expression, but rather - # use a dedicated lattice property `ThrownEscape` - continue - elseif head === :isdefined - # just returns `Bool`, nothing accounts for any usages - continue - elseif head === :enter || head === :leave || head === :pop_exception - # these exception frame managements doesn't account for any usages - # we can just ignore escape information from - continue - elseif head === :gc_preserve_begin || head === :gc_preserve_end - # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information - # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere - continue - else - add_changes!(stmt.args, ir, AllEscape(), changes) - end - elseif isa(stmt, GlobalRef) # global load - add_change!(SSAValue(pc), ir, GlobalEscape(), changes) - elseif isa(stmt, PiNode) - if isdefined(stmt, :val) - info = state.ssavalues[pc] - add_change!(stmt.val, ir, info, changes) - end - elseif isa(stmt, PhiNode) - info = state.ssavalues[pc] - values = stmt.values - for i in 1:length(values) - if isassigned(values, i) - add_change!(values[i], ir, info, changes) - end - end - elseif isa(stmt, PhiCNode) - info = state.ssavalues[pc] - values = stmt.values - for i in 1:length(values) - if isassigned(values, i) - add_change!(values[i], ir, info, changes) - end - end - elseif isa(stmt, UpsilonNode) - if isdefined(stmt, :val) - info = state.ssavalues[pc] - add_change!(stmt.val, ir, info, changes) - end - elseif isa(stmt, ReturnNode) - if isdefined(stmt, :val) - add_change!(stmt.val, ir, ReturnEscape(pc), changes) - end - else - #@assert stmt isa GotoNode || stmt isa GotoIfNot || stmt isa GlobalRef || isnothing(stmt) - continue - end - - isempty(changes) && continue - - anyupdate |= propagate_changes!(state, changes) - - empty!(changes) - end - - anyupdate || break - end - - return state -end diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 7cef7c7cb82d7..84d7ef6344b42 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1384,12 +1384,21 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end -is_array_allocation(stmt::Expr) = - is_known_fcall(stmt, - (:jl_alloc_array_1d, - :jl_alloc_array_2d, - :jl_alloc_array_3d, - :jl_new_array)) +is_array_allocation(stmt::Expr) = _is_known_fcall(stmt, ( + :jl_alloc_array_1d, + :jl_alloc_array_2d, + :jl_alloc_array_3d, + :jl_new_array)) +function _is_known_fcall(stmt::Expr, funcs) + isexpr(stmt, :foreigncall) || return false + s = stmt.args[1] + isa(s, QuoteNode) && (s = s.value) + isa(s, Symbol) || return false + for func in funcs + s === func && return true + end + return false +end function memory_opt!(ir::IRCode, escape_state) compact = IncrementalCompact(ir, false) From 4b33fc79b423abc8978d960c090c912c064b5068 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Thu, 23 Dec 2021 18:49:24 +0900 Subject: [PATCH 09/41] =?UTF-8?q?add=20simple=20`all`/`any`=20definitions?= =?UTF-8?q?=20to=20make=20`::BitSet=20=E2=8A=86=20::BitSet`=20work=20insid?= =?UTF-8?q?e=20`Core.Compiler`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- base/compiler/utilities.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index e97441495f16b..9b1106e964919 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -19,6 +19,8 @@ function _any(@nospecialize(f), a) end return false end +any(@nospecialize(f), itr) = _any(f, itr) +any(itr) = _any(identity, itr) function _all(@nospecialize(f), a) for x in a @@ -26,6 +28,8 @@ function _all(@nospecialize(f), a) end return true end +all(@nospecialize(f), itr) = _all(f, itr) +all(itr) = _all(identity, itr) function contains_is(itr, @nospecialize(x)) for y in itr From cb4b1b8e534b02841b6dd92c53f0874a175d0e39 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Thu, 23 Dec 2021 18:49:57 +0900 Subject: [PATCH 10/41] comment dead code for now --- base/compiler/ssair/passes.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index d8207396acc3d..87da18716369b 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1451,18 +1451,18 @@ function memory_opt!(ir::IRCode, escape_state) # @eval Main (ir = $ir; rev = $revisit; esc_state = $escape_state) has_no_escape(escape_state.ssavalues[id]) || continue - # We're ok to steal the memory if we don't dominate any uses - ok = true - if haskey(uses, id) - for use in uses[id] - if ssadominates(ir, domtree, idx, use) - ok = false - break - end - end - end - ok || continue - println("saved an allocation here :", stmt) + # # We're ok to steal the memory if we don't dominate any uses + # ok = true + # if haskey(uses, id) + # for use in uses[id] + # if ssadominates(ir, domtree, idx, use) + # ok = false + # break + # end + # end + # end + # ok || continue + # println("saved an allocation here :", stmt) stmt.args[1] = Core.mutating_arrayfreeze end From bca3078dab57f0b3bbcc8f1899fea7eb37040381 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 25 Dec 2021 23:54:52 +0900 Subject: [PATCH 11/41] update to latest EA --- base/compiler/EscapeAnalysis.jl | 544 ---------- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 934 ++++++++++++++++++ base/compiler/EscapeAnalysis/disjoint_set.jl | 143 +++ base/compiler/EscapeAnalysis/utils.jl | 311 ++++++ base/compiler/compiler.jl | 2 +- base/compiler/optimize.jl | 2 +- 6 files changed, 1390 insertions(+), 546 deletions(-) delete mode 100644 base/compiler/EscapeAnalysis.jl create mode 100644 base/compiler/EscapeAnalysis/EscapeAnalysis.jl create mode 100644 base/compiler/EscapeAnalysis/disjoint_set.jl create mode 100644 base/compiler/EscapeAnalysis/utils.jl diff --git a/base/compiler/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis.jl deleted file mode 100644 index ebee3ff53bc00..0000000000000 --- a/base/compiler/EscapeAnalysis.jl +++ /dev/null @@ -1,544 +0,0 @@ -baremodule EscapeAnalysis - -export - find_escapes, - has_not_analyzed, - has_no_escape, - has_return_escape, - has_thrown_escape, - has_all_escape, - can_elide_finalizer - -# analysis -# ======== - -const _TOP_MOD = ccall(:jl_base_relative_to, Any, (Any,), EscapeAnalysis)::Module - -# imports -import ._TOP_MOD: == -# usings -import Core: - MethodInstance, - Const, - Argument, - SSAValue, - PiNode, - PhiNode, - UpsilonNode, - PhiCNode, - ReturnNode, - GotoNode, - GotoIfNot, - SimpleVector -import ._TOP_MOD: # Base definitions - @eval, @assert, @nospecialize, @__MODULE__, Vector, BitSet, IdDict, - !, !==, ≠, +, -, ≤, &, |, include, error, missing, println, - ∪, ⊆, ∩, :, length, get, first, last, in, isempty, isassigned, push!, empty! -import Core.Compiler: # Core.Compiler specific definitions - IRCode, IR_FLAG_EFFECT_FREE, - isbitstype, isexpr, is_meta_expr_head, widenconst, argextype, singleton_type - -""" - x::EscapeLattice - -A lattice for escape information, which holds the following properties: -- `x.Analyzed::Bool`: not formally part of the lattice, indicates `x` has not been analyzed at all -- `x.ReturnEscape::Bool`: indicates `x` may escape to the caller via return (possibly as a field), - where `x.ReturnEscape && 0 ∈ x.EscapeSites` has the special meaning that it's visible to - the caller simply because it's passed as call argument -- `x.ThrownEscape::Bool`: indicates `x` may escape to somewhere through an exception (possibly as a field) -- `x.EscapeSites::BitSet`: records program counters (SSA numbers) where `x` can escape -- `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through `setfield!` on argument(s) - * `-1` : no escape - * `0` : unknown or multiple - * `n` : through argument N - -These attributes can be combined to create a partial lattice that has a finite height, given -that input program has a finite number of statements, which is assured by Julia's semantics. - -There are utility constructors to create common `EscapeLattice`s, e.g., -- `NoEscape()`: the bottom element of this lattice, meaning it won't escape to anywhere -- `AllEscape()`: the topmost element of this lattice, meaning it will escape to everywhere - -The escape analysis will transition these elements from the bottom to the top, -in the same direction as Julia's native type inference routine. -An abstract state will be initialized with the bottom(-like) elements: -- the call arguments are initialized as `ArgumentReturnEscape()`, because they're visible from a caller immediately -- the other states are initialized as `NotAnalyzed()`, which is a special lattice element that - is slightly lower than `NoEscape`, but at the same time doesn't represent any meaning - other than it's not analyzed yet (thus it's not formally part of the lattice). -""" -struct EscapeLattice - Analyzed::Bool - ReturnEscape::Bool - ThrownEscape::Bool - EscapeSites::BitSet - # TODO: ArgEscape::Int -end - -# precomputed default values in order to eliminate computations at each callsite -const EMPTY_ESCAPE_SITES = BitSet() -const ARGUMENT_ESCAPE_SITES = BitSet(0) - -# the constructors -NotAnalyzed() = EscapeLattice(false, false, false, EMPTY_ESCAPE_SITES) # not formally part of the lattice -NoEscape() = EscapeLattice(true, false, false, EMPTY_ESCAPE_SITES) -ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc)) -ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc)) -ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES) -let - all_escape_sites = BitSet(0:100_000) - global AllEscape() = EscapeLattice(true, true, true, all_escape_sites) - # used for `show` - global AllReturnEscape() = EscapeLattice(true, true, false, all_escape_sites) - global AllThrownEscape() = EscapeLattice(true, false, true, all_escape_sites) -end - -# Convenience names for some ⊑ queries -export - has_not_analyzed, - has_no_escape, - has_return_escape, - has_thrown_escape, - has_all_escape, - can_elide_finalizer -has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() -has_no_escape(x::EscapeLattice) = x ⊑ NoEscape() -has_return_escape(x::EscapeLattice) = x.ReturnEscape -has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites -has_thrown_escape(x::EscapeLattice) = x.ThrownEscape -has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites -has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x - -""" - can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool - -Queries the validity of the finalizer elision optimization at the `return` site of statement `pc`, -which inserts `finalize` call when the lifetime of interested object ends. -Note that we don't need to take `x.ThrownEscape` into account because it would have never -been thrown when the program execution reaches the `return` site. -""" -can_elide_finalizer(x::EscapeLattice, pc::Int) = - !(has_return_escape(x, 0) || has_return_escape(x, pc)) - -# we need to make sure this `==` operator corresponds to lattice equality rather than object equality, -# otherwise `propagate_changes` can't detect the convergence -x::EscapeLattice == y::EscapeLattice = begin - return x.Analyzed === y.Analyzed && - x.ReturnEscape === y.ReturnEscape && - x.ThrownEscape === y.ThrownEscape && - x.EscapeSites == y.EscapeSites && - true -end - -x::EscapeLattice ⊑ y::EscapeLattice = begin - if x.Analyzed ≤ y.Analyzed && - x.ReturnEscape ≤ y.ReturnEscape && - x.ThrownEscape ≤ y.ThrownEscape && - x.EscapeSites ⊆ y.EscapeSites && - true - return true - end - return false -end -x::EscapeLattice ⊏ y::EscapeLattice = x ⊑ y && !(y ⊑ x) -x::EscapeLattice ⋤ y::EscapeLattice = !(y ⊑ x) - -x::EscapeLattice ⊔ y::EscapeLattice = begin - return EscapeLattice( - x.Analyzed | y.Analyzed, - x.ReturnEscape | y.ReturnEscape, - x.ThrownEscape | y.ThrownEscape, - x.EscapeSites ∪ y.EscapeSites, - ) -end - -x::EscapeLattice ⊓ y::EscapeLattice = begin - return EscapeLattice( - x.Analyzed & y.Analyzed, - x.ReturnEscape & y.ReturnEscape, - x.ThrownEscape & y.ThrownEscape, - x.EscapeSites ∩ y.EscapeSites, - ) -end - -# TODO setup a more effient struct for cache -# which can discard escape information on SSS values and arguments that don't join dispatch signature - -""" - state::EscapeState - -Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`: -- `state.arguments::Vector{EscapeLattice}`: escape information about "arguments" – note that - "argument" can include both call arguments and slots appearing in analysis frame -- `ssavalues::Vector{EscapeLattice}`: escape information about each SSA value -""" -struct EscapeState - arguments::Vector{EscapeLattice} - ssavalues::Vector{EscapeLattice} -end -function EscapeState(nslots::Int, nargs::Int, nstmts::Int) - arguments = EscapeLattice[ - 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] - ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] - return EscapeState(arguments, ssavalues) -end - -# we preserve `IRCode` as well just for debugging purpose -const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Tuple{EscapeState,IRCode}}() -__clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) - -const Change = Pair{Union{Argument,SSAValue},EscapeLattice} -const Changes = Vector{Change} - -""" - find_escapes(ir::IRCode, nargs::Int) -> EscapeState - -Escape analysis implementation is based on the data-flow algorithm described in the paper [^MM02]. -The analysis works on the lattice of [`EscapeLattice`](@ref) and transitions lattice elements -from the bottom to the top in a _backward_ way, i.e. data flows from usage cites to definitions, -until every lattice gets converged to a fixed point by maintaining a (conceptual) working set -that contains program counters corresponding to remaining SSA statements to be analyzed. -The analysis only manages a single global state that tracks `EscapeLattice` of each argument -and SSA statement, but also note that some flow-sensitivity is encoded as program counters -recorded in the `EscapeSites` property of each each lattice element. - -[^MM02]: _A Graph-Free approach to Data-Flow Analysis_. - Markas Mohnen, 2002, April. - . -""" -function find_escapes(ir::IRCode, nargs::Int) - (; stmts, sptypes, argtypes) = ir - nstmts = length(stmts) - - # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties - state = EscapeState(length(argtypes), nargs, nstmts) - changes = Changes() # stashes changes that happen at current statement - - while true - local anyupdate = false - - for pc in nstmts:-1:1 - stmt = stmts.inst[pc] - - # we escape statements with the `ThrownEscape` property using the effect-freeness - # information computed by the inliner - is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 - - # collect escape information - if isa(stmt, Expr) - head = stmt.head - if head === :call - has_changes = escape_call!(ir, pc, stmt.args, state, changes) - if !is_effect_free - for x in stmt.args - add_change!(x, ir, ThrownEscape(pc), changes) - end - else - has_changes || continue - end - elseif head === :invoke - escape_invoke!(ir, pc, stmt.args, state, changes) - elseif head === :new || head === :splatnew - escape_new!(ir, pc, stmt.args, state, changes) - elseif head === :(=) - lhs, rhs = stmt.args - if isa(lhs, GlobalRef) # global store - add_change!(rhs, ir, AllEscape(), changes) - end - elseif head === :foreigncall - escape_foreigncall!(ir, pc, stmt.args, state, changes) - elseif head === :throw_undef_if_not # XXX when is this expression inserted ? - add_change!(stmt.args[1], ir, ThrownEscape(pc), changes) - elseif is_meta_expr_head(head) - # meta expressions doesn't account for any usages - continue - elseif head === :static_parameter - # :static_parameter refers any of static parameters, but since they exist - # statically, we're really not interested in their escapes - continue - elseif head === :copyast - # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values - continue - elseif head === :undefcheck - # undefcheck is temporarily inserted by compiler - # it will be processd be later pass so it won't change any of escape states - continue - elseif head === :the_exception - # we don't propagate escape information on exceptions via this expression, but rather - # use a dedicated lattice property `ThrownEscape` - continue - elseif head === :isdefined - # just returns `Bool`, nothing accounts for any usages - continue - elseif head === :enter || head === :leave || head === :pop_exception - # these exception frame managements doesn't account for any usages - # we can just ignore escape information from - continue - elseif head === :gc_preserve_begin || head === :gc_preserve_end - # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information - # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere - continue - else - for x in stmt.args - add_change!(x, ir, AllEscape(), changes) - end - end - elseif isa(stmt, GlobalRef) # global load - add_change!(SSAValue(pc), ir, AllEscape(), changes) - elseif isa(stmt, PiNode) - if isdefined(stmt, :val) - info = state.ssavalues[pc] - add_change!(stmt.val, ir, info, changes) - end - elseif isa(stmt, PhiNode) - escape_backedges!(ir, pc, stmt.values, state, changes) - elseif isa(stmt, PhiCNode) - escape_backedges!(ir, pc, stmt.values, state, changes) - elseif isa(stmt, UpsilonNode) - if isdefined(stmt, :val) - info = state.ssavalues[pc] - add_change!(stmt.val, ir, info, changes) - end - elseif isa(stmt, ReturnNode) - if isdefined(stmt, :val) - add_change!(stmt.val, ir, ReturnEscape(pc), changes) - end - elseif isa(stmt, SSAValue) - # NOTE after SROA, we may see SSA value as statement - info = state.ssavalues[pc] - add_change!(stmt, ir, info, changes) - else - @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me - continue - end - - isempty(changes) && continue - - anyupdate |= propagate_changes!(state, changes) - - empty!(changes) - end - - anyupdate || break - end - - return state -end - -# propagate changes, and check convergence -function propagate_changes!(state::EscapeState, changes::Changes) - local anychanged = false - - for (x, info) in changes - if isa(x, Argument) - old = state.arguments[x.n] - new = old ⊔ info - if old ≠ new - state.arguments[x.n] = new - anychanged |= true - end - else - x = x::SSAValue - old = state.ssavalues[x.id] - new = old ⊔ info - if old ≠ new - state.ssavalues[x.id] = new - anychanged |= true - end - end - end - - return anychanged -end - -function add_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) - if isa(x, Argument) || isa(x, SSAValue) - if !isbitstype(widenconst(argextype(x, ir, ir.sptypes, ir.argtypes))) - push!(changes, Change(x, info)) - end - end -end - -function escape_backedges!(ir::IRCode, pc::Int, backedges::Vector{Any}, - state::EscapeState, changes::Changes) - info = state.ssavalues[pc] - for i in 1:length(backedges) - if isassigned(backedges, i) - add_change!(backedges[i], ir, info, changes) - end - end -end - -function escape_call!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) - ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) - f = singleton_type(ft) - if isa(f, Core.IntrinsicFunction) - return false # COMBAK we may break soundness here, e.g. `pointerref` - end - result = escape_builtin!(f, ir, pc, args, state, changes) - if result === false - return false # nothing to propagate - elseif result === missing - # if this call hasn't been handled by any of pre-defined handlers, - # we escape this call conservatively - for i in 2:length(args) - add_change!(args[i], ir, AllEscape(), changes) - end - return true - else - return true - end -end - -function escape_invoke!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) - linfo = first(args)::MethodInstance - cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) - args = args[2:end] - if cache === nothing - for x in args - add_change!(x, ir, AllEscape(), changes) - end - else - (linfostate, _ #=ir::IRCode=#) = cache - retinfo = state.ssavalues[pc] # escape information imposed on the call statement - method = linfo.def::Method - nargs = Int(method.nargs) - for i in 1:length(args) - arg = args[i] - if i ≤ nargs - arginfo = linfostate.arguments[i] - else # handle isva signature: COMBAK will this invalid once we encode alias information ? - arginfo = linfostate.arguments[nargs] - end - if isempty(arginfo.ReturnEscape) - @eval Main (ir = $ir; linfo = $linfo) - error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") - end - info = from_interprocedural(arginfo, retinfo, pc) - add_change!(arg, ir, info, changes) - end - end -end - -# reinterpret the escape information imposed on the callee argument (`arginfo`) in the -# context of the caller frame using the escape information imposed on the return value (`retinfo`) -function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc::Int) - @assert arginfo.ReturnEscape - if arginfo.ThrownEscape - EscapeSites = BitSet(pc) - else - EscapeSites = EMPTY_ESCAPE_SITES - end - newarginfo = EscapeLattice(true, false, arginfo.ThrownEscape, EscapeSites) - if arginfo.EscapeSites === ARGUMENT_ESCAPE_SITES - # if this is simply passed as the call argument, we can discard the `ReturnEscape` - # information and just propagate the other escape information - return newarginfo - else - # if this can be returned, we have to merge its escape information with - # that of the current statement - return newarginfo ⊔ retinfo - end -end - -function escape_new!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - add_change!(SSAValue(pc), ir, info, changes) # we will be interested in if this allocation escapes or not - end - - # propagate the escape information of this object to all its fields as well - # since they can be accessed through the object - for i in 2:length(args) - add_change!(args[i], ir, info, changes) - end -end - -# escape every argument `(args[6:length(args[3])])` and the name `args[1]` -# TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls -function escape_foreigncall!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) - foreigncall_nargs = length((args[3])::SimpleVector) - name = args[1] - # if normalize(name) === :jl_gc_add_finalizer_th - # # add `FinalizerEscape` ? - # end - add_change!(name, ir, ThrownEscape(pc), changes) - for i in 6:5+foreigncall_nargs - add_change!(args[i], ir, ThrownEscape(pc), changes) - end -end - -# NOTE error cases will be handled in `find_escapes` anyway, so we don't need to take care of them below -# TODO implement more builtins, make them more accurate -# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? - -escape_builtin!(@nospecialize(f), _...) = return missing - -# safe builtins -escape_builtin!(::typeof(isa), _...) = return false -escape_builtin!(::typeof(typeof), _...) = return false -escape_builtin!(::typeof(Core.sizeof), _...) = return false -escape_builtin!(::typeof(===), _...) = return false -# not really safe, but `ThrownEscape` will be imposed later -escape_builtin!(::typeof(isdefined), _...) = return false -escape_builtin!(::typeof(throw), _...) = return false - -function escape_builtin!(::typeof(Core.ifelse), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) - length(args) == 4 || return - f, cond, th, el = args - info = state.ssavalues[pc] - condt = argextype(cond, ir, ir.sptypes, ir.argtypes) - if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) - if cond - add_change!(th, ir, info, changes) - else - add_change!(el, ir, info, changes) - end - else - add_change!(th, ir, info, changes) - add_change!(el, ir, info, changes) - end -end - -function escape_builtin!(::typeof(typeassert), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) - length(args) == 3 || return - f, obj, typ = args - info = state.ssavalues[pc] - add_change!(obj, ir, info, changes) -end - -function escape_builtin!(::typeof(tuple), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - end - for i in 2:length(args) - add_change!(args[i], ir, info, changes) - end -end - -# TODO don't propagate escape information to the 1st argument, but propagate information to aliased field -function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) - # only propagate info when the field itself is non-bitstype - isbitstype(widenconst(ir.stmts.type[pc])) && return true - info = state.ssavalues[pc] - if info == NotAnalyzed() - info = NoEscape() - end - for i in 2:length(args) - add_change!(args[i], ir, info, changes) - end -end - -# NOTE define fancy package utilities when developing EA as an external package -if !(_TOP_MOD === Core.Compiler) - include(@__MODULE__, "utils.jl") -end - -end # baremodule EscapeAnalysis diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl new file mode 100644 index 0000000000000..de65c97341425 --- /dev/null +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -0,0 +1,934 @@ +baremodule EscapeAnalysis + +export + find_escapes, + has_not_analyzed, + has_no_escape, + has_return_escape, + has_thrown_escape, + has_all_escape, + is_sroa_eligible, + can_elide_finalizer + +# analysis +# ======== + +const _TOP_MOD = ccall(:jl_base_relative_to, Any, (Any,), EscapeAnalysis)::Module + +# imports +import ._TOP_MOD: == +# usings +import Core: + MethodInstance, Const, Argument, SSAValue, PiNode, PhiNode, UpsilonNode, PhiCNode, + ReturnNode, GotoNode, GotoIfNot, SimpleVector +import ._TOP_MOD: # Base definitions + @__MODULE__, @eval, @assert, @nospecialize, @inbounds, @inline, @noinline, @label, @goto, + !, !==, !=, ≠, +, -, ≤, <, ≥, >, &, |, include, error, missing, + Vector, BitSet, IdDict, IdSet, ∪, ⊆, ∩, :, length, get, first, last, in, isempty, + isassigned, push!, empty!, max, min +import Core.Compiler: # Core.Compiler specific definitions + isbitstype, isexpr, is_meta_expr_head, copy, println, + IRCode, IR_FLAG_EFFECT_FREE, widenconst, argextype, singleton_type, fieldcount_noerror, + try_compute_fieldidx, hasintersect + +if isdefined(Core.Compiler, :try_compute_field) + import Core.Compiler: try_compute_field +else + function try_compute_field(ir::IRCode, @nospecialize(field)) + # fields are usually literals, handle them manually + if isa(field, QuoteNode) + field = field.value + elseif isa(field, Int) || isa(field, Symbol) + # try to resolve other constants, e.g. global reference + else + field = argextype(field, ir) + if isa(field, Const) + field = field.val + else + return nothing + end + end + return isa(field, Union{Int, Symbol}) ? field : nothing + end +end + +if _TOP_MOD !== Core.Compiler + include(@__MODULE__, "disjoint_set.jl") +else + include(@__MODULE__, "compiler/EscapeAnalysis/disjoint_set.jl") +end + +const EscapeSet = IdSet{Any} +const EscapeSets = Vector{EscapeSet} + +""" + x::EscapeLattice + +A lattice for escape information, which holds the following properties: +- `x.Analyzed::Bool`: not formally part of the lattice, indicates `x` has not been analyzed at all +- `x.ReturnEscape::Bool`: indicates `x` may escape to the caller via return, + where `x.ReturnEscape && 0 ∈ x.EscapeSites` has the special meaning that it's visible to + the caller simply because it's passed as call argument +- `x.ThrownEscape::Bool`: indicates `x` may escape to somewhere through an exception +- `x.EscapeSites::BitSet`: records SSA statements where `x` can escape via any of + `ReturnEscape` or `ThrownEscape` +- `x.FieldEscapes::Union{Vector{IdSet{Any}},Bool}`: maintains all possible values that impose + escape information on fields of `x`: + * `x.FieldEscapes === false` indicates the fields of `x` isn't analyzed yet + * `x.FieldEscapes === true` indicates the fields of `x` can't be analyzed, e.g. the type of `x` + is not known or is not concrete and thus its fields can't be known precisely + * otherwise `x.FieldEscapes::Vector{IdSet{Any}}` holds all the possible values that can escape + fields of `x`, which allows EA to propagate propagate escape information imposed on a field + of `x` to its values (by analyzing `Expr(:new, ...)` and `setfield!(x, ...)`). +- `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through + `setfield!` on argument(s) + * `-1` : no escape + * `0` : unknown or multiple + * `n` : through argument N + +There are utility constructors to create common `EscapeLattice`s, e.g., +- `NoEscape()`: the bottom element of this lattice, meaning it won't escape to anywhere +- `AllEscape()`: the topmost element of this lattice, meaning it will escape to everywhere + +`find_escapes` will transition these elements from the bottom to the top, +in the same direction as Julia's native type inference routine. +An abstract state will be initialized with the bottom(-like) elements: +- the call arguments are initialized as `ArgumentReturnEscape()`, because they're visible from a caller immediately +- the other states are initialized as `NotAnalyzed()`, which is a special lattice element that + is slightly lower than `NoEscape`, but at the same time doesn't represent any meaning + other than it's not analyzed yet (thus it's not formally part of the lattice). +""" +struct EscapeLattice + Analyzed::Bool + ReturnEscape::Bool + ThrownEscape::Bool + EscapeSites::BitSet + FieldEscapes::Union{EscapeSets,Bool} + # TODO: ArgEscape::Int + + function EscapeLattice(Analyzed::Bool, + ReturnEscape::Bool, + ThrownEscape::Bool, + EscapeSites::BitSet, + FieldEscapes::Union{EscapeSets,Bool}, + ) + @nospecialize FieldEscapes + return new( + Analyzed, + ReturnEscape, + ThrownEscape, + EscapeSites, + FieldEscapes, + ) + end + function EscapeLattice(x::EscapeLattice, + # non-concrete fields should be passed as default arguments + # in order to avoid allocating non-concrete `NamedTuple`s + FieldEscapes::Union{EscapeSets,Bool} = x.FieldEscapes; + Analyzed::Bool = x.Analyzed, + ReturnEscape::Bool = x.ReturnEscape, + ThrownEscape::Bool = x.ThrownEscape, + EscapeSites::BitSet = x.EscapeSites, + ) + @nospecialize FieldEscapes + return new( + Analyzed, + ReturnEscape, + ThrownEscape, + EscapeSites, + FieldEscapes, + ) + end +end + +# precomputed default values in order to eliminate computations at each callsite +const BOT_ESCAPE_SITES = BitSet() +const ARGUMENT_ESCAPE_SITES = BitSet(0) +const TOP_ESCAPE_SITES = BitSet(0:100_000) + +const BOT_FIELD_SETS = false +const TOP_FIELD_SETS = true + +# the constructors +NotAnalyzed() = EscapeLattice(false, false, false, BOT_ESCAPE_SITES, BOT_FIELD_SETS) # not formally part of the lattice +NoEscape() = EscapeLattice(true, false, false, BOT_ESCAPE_SITES, BOT_FIELD_SETS) +ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc), BOT_FIELD_SETS) +ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc), BOT_FIELD_SETS) +ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES, TOP_FIELD_SETS) # TODO allow interprocedural field analysis? +AllEscape() = EscapeLattice(true, true, true, TOP_ESCAPE_SITES, TOP_FIELD_SETS) + +# Convenience names for some ⊑ queries +has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() +has_no_escape(x::EscapeLattice) = ignore_fieldsets(x) ⊑ NoEscape() +has_return_escape(x::EscapeLattice) = x.ReturnEscape +has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites +has_thrown_escape(x::EscapeLattice) = x.ThrownEscape +has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites +has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x + +ignore_fieldsets(x::EscapeLattice) = EscapeLattice(x, BOT_FIELD_SETS) +has_fieldsets(x::EscapeLattice) = !isa(x.FieldEscapes, Bool) + +# TODO is_sroa_eligible: consider throwness? + +""" + is_sroa_eligible(x::EscapeLattice) -> Bool + +Queries allocation eliminability by SROA. +""" +is_sroa_eligible(x::EscapeLattice) = x.FieldEscapes !== TOP_FIELD_SETS && !has_return_escape(x) + +""" + can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool + +Queries the validity of the finalizer elision optimization at the return site of SSA statement `pc`, +which inserts `finalize` call when the lifetime of interested object ends. +Note that we don't need to take `x.ThrownEscape` into account because it would have never +been thrown when the program execution reaches the `return` site. +""" +can_elide_finalizer(x::EscapeLattice, pc::Int) = + !(has_return_escape(x, 0) || has_return_escape(x, pc)) + +# we need to make sure this `==` operator corresponds to lattice equality rather than object equality, +# otherwise `propagate_changes` can't detect the convergence +x::EscapeLattice == y::EscapeLattice = begin + xf, yf = x.FieldEscapes, y.FieldEscapes + if isa(xf, Bool) + isa(yf, Bool) || return false + xf === yf || return false + else + isa(yf, Bool) && return false + xf == yf || return false + end + return x.Analyzed === y.Analyzed && + x.ReturnEscape === y.ReturnEscape && + x.ThrownEscape === y.ThrownEscape && + x.EscapeSites == y.EscapeSites && + true +end + +""" + x::EscapeLattice ⊑ y::EscapeLattice -> Bool + +The non-strict partial order over `EscapeLattice`. +""" +x::EscapeLattice ⊑ y::EscapeLattice = begin + xf, yf = x.FieldEscapes, y.FieldEscapes + if isa(xf, Bool) + xf && yf !== true && return false + else + if isa(yf, Bool) + yf === false && return false + else + xf, yf = xf::EscapeSets, yf::EscapeSets + xn, yn = length(xf), length(yf) + xn > yn && return false + for i in 1:xn + xf[i] ⊆ yf[i] || return false + end + end + end + if x.Analyzed ≤ y.Analyzed && + x.ReturnEscape ≤ y.ReturnEscape && + x.ThrownEscape ≤ y.ThrownEscape && + x.EscapeSites ⊆ y.EscapeSites && + true + return true + end + return false +end + +""" + x::EscapeLattice ⊏ y::EscapeLattice -> Bool + +The strict partial order over `EscapeLattice`. +This is defined as the irreflexive kernel of `⊏`. +""" +x::EscapeLattice ⊏ y::EscapeLattice = x ⊑ y && !(y ⊑ x) + +""" + x::EscapeLattice ⋤ y::EscapeLattice -> Bool + +This order could be used as a slightly more efficient version of the strict order `⊏`, +where we can safely assume `x ⊑ y` holds. +""" +x::EscapeLattice ⋤ y::EscapeLattice = !(y ⊑ x) + +""" + x::EscapeLattice ⊔ y::EscapeLattice -> EscapeLattice + +Computes the join of `x` and `y` in the partial order defined by `EscapeLattice`. +""" +x::EscapeLattice ⊔ y::EscapeLattice = begin + xf, yf = x.FieldEscapes, y.FieldEscapes + if xf === true || yf === true + FieldEscapes = true + elseif xf === false + FieldEscapes = yf + elseif yf === false + FieldEscapes = xf + else + xf, yf = xf::EscapeSets, yf::EscapeSets + xn, yn = length(xf), length(yf) + nmax, nmin = max(xn, yn), min(xn, yn) + FieldEscapes = EscapeSets(undef, nmax) + for i in 1:nmax + if i > nmin + FieldEscapes[i] = (xn > yn ? xf : yf)[i] + else + FieldEscapes[i] = xf[i] ∪ yf[i] + end + end + end + # try to avoid new allocations as minor optimizations + xe, ye = x.EscapeSites, y.EscapeSites + if xe === TOP_ESCAPE_SITES || ye === TOP_ESCAPE_SITES + EscapeSites = TOP_ESCAPE_SITES + elseif xe === BOT_ESCAPE_SITES + EscapeSites = ye + elseif ye === BOT_ESCAPE_SITES + EscapeSites = xe + else + EscapeSites = xe ∪ ye + end + return EscapeLattice( + x.Analyzed | y.Analyzed, + x.ReturnEscape | y.ReturnEscape, + x.ThrownEscape | y.ThrownEscape, + EscapeSites, + FieldEscapes, + ) +end + +""" + x::EscapeLattice ⊓ y::EscapeLattice -> EscapeLattice + +Computes the meet of `x` and `y` in the partial order defined by `EscapeLattice`. +""" +x::EscapeLattice ⊓ y::EscapeLattice = begin + return EscapeLattice( + x.Analyzed & y.Analyzed, + x.ReturnEscape & y.ReturnEscape, + x.ThrownEscape & y.ThrownEscape, + x.EscapeSites ∩ y.EscapeSites, + x.FieldEscapes, # FIXME + ) +end + +# TODO setup a more effient struct for cache +# which can discard escape information on SSS values and arguments that don't join dispatch signature + +""" + state::EscapeState + +Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`: +- `state.arguments::Vector{EscapeLattice}`: escape information about "arguments"; + note that "argument" can include both call arguments and slots appearing in analysis frame +- `ssavalues::Vector{EscapeLattice}`: escape information about each SSA value +- `aliaset::IntDisjointSet{Int}`: a disjoint set that maintains aliased arguments and SSA values +""" +struct EscapeState + arguments::Vector{EscapeLattice} + ssavalues::Vector{EscapeLattice} + aliasset::IntDisjointSet{Int} +end +function EscapeState(nslots::Int, nargs::Int, nstmts::Int) + arguments = EscapeLattice[ + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] + ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] + aliaset = AliasSet(nslots+nstmts) + return EscapeState(arguments, ssavalues, aliaset) +end + +const AliasSet = IntDisjointSet{Int} +function alias_idx(@nospecialize(x), ir::IRCode) + if isa(x, Argument) + return x.n + elseif isa(x, SSAValue) + return x.id + length(ir.argtypes) + else + return nothing + end +end +function alias_val(idx::Int, ir::IRCode) + n = length(ir.argtypes) + return idx > n ? SSAValue(idx-n) : Argument(idx) +end +function get_aliases(aliasset::AliasSet, @nospecialize(key), ir::IRCode) + idx = alias_idx(key, ir) + idx === nothing && return nothing + root = find_root!(aliasset, idx) + if idx ≠ root || aliasset.ranks[idx] > 0 + # the size of this alias set containing `key` is larger than 1, + # collect the entire alias set + aliases = Union{Argument,SSAValue}[] + for i in 1:length(aliasset.parents) + if aliasset.parents[i] == root + push!(aliases, alias_val(i, ir)) + end + end + return aliases + else + return nothing + end +end + +# we preserve `IRCode` as well just for debugging purpose +const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Tuple{EscapeState,IRCode}}() +__clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) + +const EscapeChange = Pair{Union{Argument,SSAValue},EscapeLattice} +const AliasChange = Pair{Int,Int} +const Changes = Vector{Union{EscapeChange,AliasChange}} + +""" + find_escapes(ir::IRCode, nargs::Int) -> EscapeState + +Analyzes escape information in `ir`. +`nargs` is the number of actual arguments of the analyzed call. +""" +function find_escapes(ir::IRCode, nargs::Int) + (; stmts, sptypes, argtypes) = ir + nstmts = length(stmts) + + # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties + state = EscapeState(length(argtypes), nargs, nstmts) + changes = Changes() # stashes changes that happen at current statement + + local debug_itr_counter = 0 + while true + local anyupdate = false + + for pc in nstmts:-1:1 + stmt = stmts.inst[pc] + + # we escape statements with the `ThrownEscape` property using the effect-freeness + # information computed by the inliner + is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 + + # collect escape information + if isa(stmt, Expr) + head = stmt.head + if head === :call + has_changes = escape_call!(ir, pc, stmt.args, state, changes) + # TODO throwness ≠ "effect-free-ness" + if !is_effect_free + for x in stmt.args + add_escape_change!(x, ir, ThrownEscape(pc), changes) + end + else + has_changes || continue + end + elseif head === :invoke + escape_invoke!(ir, pc, stmt.args, state, changes) + elseif head === :new || head === :splatnew + escape_new!(ir, pc, stmt.args, state, changes) + elseif head === :(=) + lhs, rhs = stmt.args + if isa(lhs, GlobalRef) # global store + add_escape_change!(rhs, ir, AllEscape(), changes) + else + invalid_escape_assignment!(ir, pc) + end + elseif head === :foreigncall + escape_foreigncall!(ir, pc, stmt.args, state, changes) + elseif head === :throw_undef_if_not # XXX when is this expression inserted ? + add_escape_change!(stmt.args[1], ir, ThrownEscape(pc), changes) + elseif is_meta_expr_head(head) + # meta expressions doesn't account for any usages + continue + elseif head === :static_parameter + # :static_parameter refers any of static parameters, but since they exist + # statically, we're really not interested in their escapes + continue + elseif head === :copyast + # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values + continue + elseif head === :undefcheck + # undefcheck is temporarily inserted by compiler + # it will be processd be later pass so it won't change any of escape states + continue + elseif head === :the_exception + # we don't propagate escape information on exceptions via this expression, but rather + # use a dedicated lattice property `ThrownEscape` + continue + elseif head === :isdefined + # just returns `Bool`, nothing accounts for any usages + continue + elseif head === :enter || head === :leave || head === :pop_exception + # these exception frame managements doesn't account for any usages + # we can just ignore escape information from + continue + elseif head === :gc_preserve_begin || head === :gc_preserve_end + # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information + # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere + continue + else + for x in stmt.args + add_escape_change!(x, ir, AllEscape(), changes) + end + end + elseif isa(stmt, ReturnNode) + if isdefined(stmt, :val) + add_escape_change!(stmt.val, ir, ReturnEscape(pc), changes) + end + elseif isa(stmt, PhiNode) + escape_edges!(ir, pc, stmt.values, state, changes) + elseif isa(stmt, PiNode) + escape_val!(ir, pc, stmt, state, changes) + elseif isa(stmt, PhiCNode) + escape_edges!(ir, pc, stmt.values, state, changes) + elseif isa(stmt, UpsilonNode) + escape_val!(ir, pc, stmt, state, changes) + elseif isa(stmt, GlobalRef) # global load + add_escape_change!(SSAValue(pc), ir, AllEscape(), changes) + elseif isa(stmt, SSAValue) + # NOTE after SROA, we may see SSA value as statement + info = state.ssavalues[pc] + add_escape_change!(stmt, ir, info, changes) + add_alias_change!(stmt, SSAValue(pc), ir, changes) + else + @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me + continue + end + + isempty(changes) && continue + + anyupdate |= propagate_changes!(state, changes, ir) + + empty!(changes) + end + + debug_itr_counter += 1 + + anyupdate || break + end + + # if debug_itr_counter > 2 + # println("[EA] excessive iteration count found ", debug_itr_counter, " (", singleton_type(ir.argtypes[1]), ")") + # end + + return state +end + +# propagate changes, and check convergence +function propagate_changes!(state::EscapeState, changes::Changes, ir::IRCode) + local anychanged = false + for change in changes + if isa(change, EscapeChange) + anychanged |= propagate_escape_change!(state, change) + x, info = change + aliases = get_aliases(state.aliasset, x, ir) + if aliases !== nothing + for alias in aliases + morechange = EscapeChange(alias, info) + anychanged |= propagate_escape_change!(state, morechange) + end + end + else + anychanged |= propagate_alias_change!(state, change) + end + end + return anychanged +end + +function propagate_escape_change!(state::EscapeState, change::EscapeChange) + x, info = change + if isa(x, Argument) + old = state.arguments[x.n] + new = old ⊔ info + if old ≠ new + state.arguments[x.n] = new + return true + end + else + x = x::SSAValue + old = state.ssavalues[x.id] + new = old ⊔ info + if old ≠ new + state.ssavalues[x.id] = new + return true + end + end + return false +end + +function propagate_alias_change!(state::EscapeState, change::AliasChange) + x, y = change + xroot = find_root!(state.aliasset, x) + yroot = find_root!(state.aliasset, y) + if xroot ≠ yroot + union!(state.aliasset, xroot, yroot) + return true + end + return false +end + +function add_escape_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) + if isa(x, Argument) || isa(x, SSAValue) + if !isbitstype(widenconst(argextype(x, ir))) + push!(changes, EscapeChange(x, info)) + end + end +end + +function add_alias_change!(@nospecialize(x), @nospecialize(y), ir::IRCode, changes::Changes) + xidx = alias_idx(x, ir) + yidx = alias_idx(y, ir) + if xidx !== nothing && yidx !== nothing + push!(changes, AliasChange(xidx, yidx)) + end +end + +function escape_edges!(ir::IRCode, pc::Int, edges::Vector{Any}, + state::EscapeState, changes::Changes) + info = state.ssavalues[pc] + for i in 1:length(edges) + if isassigned(edges, i) + v = edges[i] + add_escape_change!(v, ir, info, changes) + add_alias_change!(SSAValue(pc), v, ir, changes) + end + end +end + +function escape_val!(ir::IRCode, pc::Int, x, state::EscapeState, changes::Changes) + if isdefined(x, :val) + info = state.ssavalues[pc] + add_escape_change!(x.val, ir, info, changes) + add_alias_change!(SSAValue(pc), x.val, ir, changes) + end +end + +# NOTE if we don't maintain the alias set that is separated from the lattice state, we can do +# soemthing like below: it essentially incorporates forward escape propagation in our default +# backward propagation, and leads to inefficient convergence that requires more iterations +# # lhs = rhs: propagate escape information of `rhs` to `lhs` +# function escape_alias!(@nospecialize(lhs), @nospecialize(rhs), +# ir::IRCode, state::EscapeState, changes::Changes) +# if isa(rhs, SSAValue) +# vinfo = state.ssavalues[rhs.id] +# elseif isa(rhs, Argument) +# vinfo = state.arguments[rhs.n] +# else +# return +# end +# add_escape_change!(lhs, ir, vinfo, changes) +# end + +function escape_invoke!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + linfo = first(args)::MethodInstance + cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) + args = args[2:end] + if cache === nothing + for x in args + add_escape_change!(x, ir, AllEscape(), changes) + end + else + (linfostate, #=, ir::IRCode=#) = cache + retinfo = state.ssavalues[pc] # escape information imposed on the call statement + method = linfo.def::Method + nargs = Int(method.nargs) + for i in 1:length(args) + arg = args[i] + if i ≤ nargs + arginfo = linfostate.arguments[i] + else # handle isva signature: COMBAK will this be invalid once we take alias information into account ? + arginfo = linfostate.arguments[nargs] + end + isempty(arginfo.ReturnEscape) && invalid_escape_invoke!(ir, linfo) + info = from_interprocedural(arginfo, retinfo, pc) + add_escape_change!(arg, ir, info, changes) + end + end +end + +# reinterpret the escape information imposed on the callee argument (`arginfo`) in the +# context of the caller frame using the escape information imposed on the return value (`retinfo`) +function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc::Int) + @assert arginfo.ReturnEscape + if arginfo.ThrownEscape + EscapeSites = BitSet(pc) + else + EscapeSites = BOT_ESCAPE_SITES + end + newarginfo = EscapeLattice( + #=Analyzed=#true, #=ReturnEscape=#false, arginfo.ThrownEscape, EscapeSites, + # FIXME implement interprocedural effect-analysis + # currently, this essentially disables the entire field analysis + # it might be okay from the SROA point of view, since we can't remove the allocation + # as far as it's passed to a callee anyway, but still we may want some field analysis + # in order to stack allocate it + TOP_FIELD_SETS) + if arginfo.EscapeSites === ARGUMENT_ESCAPE_SITES + # if this is simply passed as the call argument, we can discard the `ReturnEscape` + # information and just propagate the other escape information + return newarginfo + else + # if this can be returned, we have to merge its escape information with + # that of the current statement + return newarginfo ⊔ retinfo + end +end + +@noinline function invalid_escape_invoke!(ir::IRCode, linfo::MethodInstance) + @eval Main (ir = $ir; linfo = $linfo) + error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") +end + +@noinline function invalid_escape_assignment!(ir::IRCode, pc::Int) + @eval Main (ir = $ir; pc = $pc) + error("unexpected assignment found: inspect `Main.pc` and `Main.pc`") +end + +function escape_new!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + objinfo = state.ssavalues[pc] + if objinfo == NotAnalyzed() + objinfo = NoEscape() + end + FieldEscapes = objinfo.FieldEscapes + nargs = length(args) + if isa(FieldEscapes, Bool) + # the fields couldn't be analyzed precisely: directly propagate the escape information + # of this object to all its fields (which is the most conservative option) + for i in 2:nargs + add_escape_change!(args[i], ir, objinfo, changes) + end + else + # fields are known: propagate escape information imposed on recorded possibilities + nf = length(FieldEscapes) + for i in 2:nargs + i-1 > nf && break # may happen when e.g. ϕ-node merges values with different types + escape_field!(args[i], FieldEscapes[i-1], ir, state, changes) + end + end +end + +function escape_field!(@nospecialize(v), FieldEscape::EscapeSet, ir::IRCode, state::EscapeState, changes::Changes) + for x in FieldEscape + if isa(x, SSAValue) + add_escape_change!(v, ir, state.ssavalues[x.id], changes) + elseif isa(x, Argument) + add_escape_change!(v, ir, state.arguments[x.n], changes) + else + continue + end + add_alias_change!(v, x, ir, changes) + end +end + +# escape every argument `(args[6:length(args[3])])` and the name `args[1]` +# TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls +function escape_foreigncall!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + foreigncall_nargs = length((args[3])::SimpleVector) + name = args[1] + # if normalize(name) === :jl_gc_add_finalizer_th + # # add `FinalizerEscape` ? + # end + add_escape_change!(name, ir, ThrownEscape(pc), changes) + for i in 6:5+foreigncall_nargs + add_escape_change!(args[i], ir, ThrownEscape(pc), changes) + end +end + +# NOTE error cases will be handled in `find_escapes` anyway, so we don't need to take care of them below +# TODO implement more builtins, make them more accurate +# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? + +function escape_call!(ir::IRCode, pc::Int, args::Vector{Any}, + state::EscapeState, changes::Changes) + ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) + f = singleton_type(ft) + if isa(f, Core.IntrinsicFunction) + return false # COMBAK we may break soundness here, e.g. `pointerref` + end + result = escape_builtin!(f, ir, pc, args, state, changes) + if result === false + return false # nothing to propagate + elseif result === missing + # if this call hasn't been handled by any of pre-defined handlers, + # we escape this call conservatively + for i in 2:length(args) + add_escape_change!(args[i], ir, AllEscape(), changes) + end + return true + else + return true + end +end + +escape_builtin!(@nospecialize(f), _...) = return missing + +# safe builtins +escape_builtin!(::typeof(isa), _...) = return false +escape_builtin!(::typeof(typeof), _...) = return false +escape_builtin!(::typeof(Core.sizeof), _...) = return false +escape_builtin!(::typeof(===), _...) = return false +# not really safe, but `ThrownEscape` will be imposed later +escape_builtin!(::typeof(isdefined), _...) = return false +escape_builtin!(::typeof(throw), _...) = return false + +function escape_builtin!(::typeof(Core.ifelse), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) == 4 || return nothing + f, cond, th, el = args + info = state.ssavalues[pc] + condt = argextype(cond, ir) + ret = SSAValue(pc) + if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) + if cond + add_escape_change!(th, ir, info, changes) + add_alias_change!(th, ret, ir, changes) + else + add_escape_change!(el, ir, info, changes) + add_alias_change!(el, ret, ir, changes) + end + else + add_escape_change!(th, ir, info, changes) + add_escape_change!(el, ir, info, changes) + add_alias_change!(th, ret, ir, changes) + add_alias_change!(el, ret, ir, changes) + end + return nothing +end + +function escape_builtin!(::typeof(typeassert), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) == 3 || return nothing + f, obj, typ = args + info = state.ssavalues[pc] + add_escape_change!(obj, ir, info, changes) + add_alias_change!(SSAValue(pc), obj, ir, changes) + return nothing +end + +function escape_builtin!(::typeof(tuple), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + escape_new!(ir, pc, args, state, changes) + return nothing +end + +function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) ≥ 3 || return nothing + obj = args[2] + typ = widenconst(argextype(obj, ir)) + if hasintersect(typ, Module) # global load + add_escape_change!(SSAValue(pc), ir, AllEscape(), changes) + end + if isa(obj, SSAValue) + objinfo = state.ssavalues[obj.id] + elseif isa(obj, Argument) + objinfo = state.arguments[obj.n] + else + return + end + FieldEscapes = objinfo.FieldEscapes + if isa(FieldEscapes, Bool) + if !FieldEscapes + # the fields of this object aren't analyzed yet: analyze them now + nfields = fieldcount_noerror(typ) + if nfields !== nothing + FieldEscapes = EscapeSet[EscapeSet() for _ in 1:nfields] + @goto add_field_escape + end + end + # the field couldn't be analyzed precisely: directly propagate the escape information + # imposed on the return value of this `getfield` call to the object (which is the most conservative option) + # but also with updated field information + ssainfo = state.ssavalues[pc] + if ssainfo == NotAnalyzed() + ssainfo = NoEscape() + end + add_escape_change!(obj, ir, EscapeLattice(ssainfo, TOP_FIELD_SETS), changes) + else + # fields are known: record the return value of this `getfield` call as a possibility that imposes escape + FieldEscapes = copy(FieldEscapes) + @label add_field_escape + if isa(typ, DataType) + fld = args[3] + fldval = try_compute_field(ir, fld) + fidx = try_compute_fieldidx(typ, fldval) + else + fidx = nothing + end + if fidx !== nothing + # the field is known precisely: propagate this escape information to the field + push!(FieldEscapes[fidx], SSAValue(pc)) + else + # the field isn't known precisely: propagate this escape information to all the fields + for FieldEscape in FieldEscapes + push!(FieldEscape, SSAValue(pc)) + end + end + add_escape_change!(obj, ir, EscapeLattice(objinfo, FieldEscapes), changes) + end + return nothing +end + +function escape_builtin!(::typeof(setfield!), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) + length(args) ≥ 4 || return nothing + obj, fld, val = args[2:4] + if isa(obj, SSAValue) + objinfo = state.ssavalues[obj.id] + elseif isa(obj, Argument) + objinfo = state.arguments[obj.n] + else + # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively + add_escape_change!(val, ir, AllEscape(), changes) + return + end + FieldEscapes = objinfo.FieldEscapes + if isa(FieldEscapes, Bool) + if !FieldEscapes + # the fields of this object aren't analyzed yet: analyze them now + typ = widenconst(argextype(obj, ir)) + nfields = fieldcount_noerror(typ) + if nfields !== nothing + # unsuccessful field analysis: update obj's escape information with new field information + FieldEscapes = EscapeSet[EscapeSet() for _ in 1:nfields] + objinfo = EscapeLattice(objinfo, FieldEscapes) + add_escape_change!(obj, ir, objinfo, changes) + @goto add_field_escape + end + # unsuccessful field analysis: update obj's escape information with new field information + objinfo = EscapeLattice(objinfo, TOP_FIELD_SETS) + add_escape_change!(obj, ir, objinfo, changes) + end + # the field couldn't be analyzed precisely: directly propagate the escape information + # of this object to the field (which is the most conservative option) + add_escape_change!(val, ir, objinfo, changes) + else + # fields are known: propagate escape information imposed on recorded possibilities + typ = widenconst(argextype(obj, ir)) + @label add_field_escape + if isa(typ, DataType) + fldval = try_compute_field(ir, fld) + fidx = try_compute_fieldidx(typ, fldval) + else + fidx = nothing + end + if fidx !== nothing + # the field is known precisely: propagate this escape information to the field + escape_field!(val, FieldEscapes[fidx], ir, state, changes) + else + # the field isn't known precisely: propagate this escape information to all the fields + for FieldEscape in FieldEscapes + escape_field!(val, FieldEscape, ir, state, changes) + end + end + end + # also propagate escape information imposed on the return value of this `setfield!` + ssainfo = state.ssavalues[pc] + if ssainfo == NotAnalyzed() + ssainfo = NoEscape() + end + add_escape_change!(val, ir, ssainfo, changes) + return nothing +end + +# NOTE define fancy package utilities when developing EA as an external package +if _TOP_MOD !== Core.Compiler + include(@__MODULE__, "utils.jl") +end + +end # baremodule EscapeAnalysis diff --git a/base/compiler/EscapeAnalysis/disjoint_set.jl b/base/compiler/EscapeAnalysis/disjoint_set.jl new file mode 100644 index 0000000000000..915bc214d7c3c --- /dev/null +++ b/base/compiler/EscapeAnalysis/disjoint_set.jl @@ -0,0 +1,143 @@ +# A disjoint set implementation adapted from +# https://github.com/JuliaCollections/DataStructures.jl/blob/f57330a3b46f779b261e6c07f199c88936f28839/src/disjoint_set.jl +# under the MIT license: https://github.com/JuliaCollections/DataStructures.jl/blob/master/License.md + +# imports +import ._TOP_MOD: + length, + eltype, + union!, + push! +# usings +import ._TOP_MOD: + OneTo, collect, zero, zeros, one, typemax + +# Disjoint-Set + +############################################################ +# +# A forest of disjoint sets of integers +# +# Since each element is an integer, we can use arrays +# instead of dictionary (for efficiency) +# +# Disjoint sets over other key types can be implemented +# based on an IntDisjointSet through a map from the key +# to an integer index +# +############################################################ + +_intdisjointset_bounds_err_msg(T) = "the maximum number of elements in IntDisjointSet{$T} is $(typemax(T))" + +""" + IntDisjointSet{T<:Integer}(n::Integer) + +A forest of disjoint sets of integers, which is a data structure +(also called a union–find data structure or merge–find set) +that tracks a set of elements partitioned +into a number of disjoint (non-overlapping) subsets. +""" +mutable struct IntDisjointSet{T<:Integer} + parents::Vector{T} + ranks::Vector{T} + ngroups::T +end + +IntDisjointSet(n::T) where {T<:Integer} = IntDisjointSet{T}(collect(OneTo(n)), zeros(T, n), n) +IntDisjointSet{T}(n::Integer) where {T<:Integer} = IntDisjointSet{T}(collect(OneTo(T(n))), zeros(T, T(n)), T(n)) +length(s::IntDisjointSet) = length(s.parents) + +""" + num_groups(s::IntDisjointSet) + +Get a number of groups. +""" +num_groups(s::IntDisjointSet) = s.ngroups +eltype(::Type{IntDisjointSet{T}}) where {T<:Integer} = T + +# find the root element of the subset that contains x +# path compression is implemented here +function find_root_impl!(parents::Vector{T}, x::Integer) where {T<:Integer} + p = parents[x] + @inbounds if parents[p] != p + parents[x] = p = _find_root_impl!(parents, p) + end + return p +end + +# unsafe version of the above +function _find_root_impl!(parents::Vector{T}, x::Integer) where {T<:Integer} + @inbounds p = parents[x] + @inbounds if parents[p] != p + parents[x] = p = _find_root_impl!(parents, p) + end + return p +end + +""" + find_root!(s::IntDisjointSet{T}, x::T) + +Find the root element of the subset that contains an member `x`. +Path compression happens here. +""" +find_root!(s::IntDisjointSet{T}, x::T) where {T<:Integer} = find_root_impl!(s.parents, x) + +""" + in_same_set(s::IntDisjointSet{T}, x::T, y::T) + +Returns `true` if `x` and `y` belong to the same subset in `s`, and `false` otherwise. +""" +in_same_set(s::IntDisjointSet{T}, x::T, y::T) where {T<:Integer} = find_root!(s, x) == find_root!(s, y) + +""" + union!(s::IntDisjointSet{T}, x::T, y::T) + +Merge the subset containing `x` and that containing `y` into one +and return the root of the new set. +""" +function union!(s::IntDisjointSet{T}, x::T, y::T) where {T<:Integer} + parents = s.parents + xroot = find_root_impl!(parents, x) + yroot = find_root_impl!(parents, y) + return xroot != yroot ? root_union!(s, xroot, yroot) : xroot +end + +""" + root_union!(s::IntDisjointSet{T}, x::T, y::T) + +Form a new set that is the union of the two sets whose root elements are +`x` and `y` and return the root of the new set. +Assume `x ≠ y` (unsafe). +""" +function root_union!(s::IntDisjointSet{T}, x::T, y::T) where {T<:Integer} + parents = s.parents + rks = s.ranks + @inbounds xrank = rks[x] + @inbounds yrank = rks[y] + + if xrank < yrank + x, y = y, x + elseif xrank == yrank + rks[x] += one(T) + end + @inbounds parents[y] = x + s.ngroups -= one(T) + return x +end + +""" + push!(s::IntDisjointSet{T}) + +Make a new subset with an automatically chosen new element `x`. +Returns the new element. Throw an `ArgumentError` if the +capacity of the set would be exceeded. +""" +function push!(s::IntDisjointSet{T}) where {T<:Integer} + l = length(s) + l < typemax(T) || throw(ArgumentError(_intdisjointset_bounds_err_msg(T))) + x = l + one(T) + push!(s.parents, x) + push!(s.ranks, zero(T)) + s.ngroups += one(T) + return x +end diff --git a/base/compiler/EscapeAnalysis/utils.jl b/base/compiler/EscapeAnalysis/utils.jl new file mode 100644 index 0000000000000..a792e9fb9b4d9 --- /dev/null +++ b/base/compiler/EscapeAnalysis/utils.jl @@ -0,0 +1,311 @@ +module EAUtils + +import ..EscapeAnalysis: EscapeAnalysis +const EA = EscapeAnalysis +const CC = Core.Compiler + +let + README = normpath(dirname(@__DIR__), "README.md") + include_dependency(README) + @doc read(README, String) EA +end + +let __init_hooks__ = [] + global __init__() = foreach(f->f(), __init_hooks__) + global register_init_hook!(@nospecialize(f)) = push!(__init_hooks__, f) +end + +# entries +# ------- + +using InteractiveUtils + +macro analyze_escapes(ex0...) + return InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :analyze_escapes, ex0) +end + +function analyze_escapes(@nospecialize(f), @nospecialize(types=Tuple{}); + world = get_world_counter(), + interp = Core.Compiler.NativeInterpreter(world)) + interp = EscapeAnalyzer(interp) + results = code_typed(f, types; optimize=true, world, interp) + isone(length(results)) || throw(ArgumentError("`analyze_escapes` only supports single analysis result")) + return EscapeResult(interp.ir, interp.state, interp.linfo) +end + +# AbstractInterpreter +# ------------------- + +# imports +import .CC: + AbstractInterpreter, + NativeInterpreter, + WorldView, + WorldRange, + InferenceParams, + OptimizationParams, + get_world_counter, + get_inference_cache, + lock_mi_inference, + unlock_mi_inference, + add_remark!, + may_optimize, + may_compress, + may_discard_trees, + verbose_stmt_info, + code_cache, + get_inference_cache +# usings +import Core: + CodeInstance, MethodInstance +import .CC: + OptimizationState, IRCode +import .EA: + find_escapes, GLOBAL_ESCAPE_CACHE + +mutable struct EscapeAnalyzer{State} <: AbstractInterpreter + native::NativeInterpreter + ir::IRCode + state::State + linfo::MethodInstance + EscapeAnalyzer(native::NativeInterpreter) = new{EscapeState}(native) +end + +CC.InferenceParams(interp::EscapeAnalyzer) = InferenceParams(interp.native) +CC.OptimizationParams(interp::EscapeAnalyzer) = OptimizationParams(interp.native) +CC.get_world_counter(interp::EscapeAnalyzer) = get_world_counter(interp.native) + +CC.lock_mi_inference(::EscapeAnalyzer, ::MethodInstance) = nothing +CC.unlock_mi_inference(::EscapeAnalyzer, ::MethodInstance) = nothing + +CC.add_remark!(interp::EscapeAnalyzer, sv, s) = add_remark!(interp.native, sv, s) + +CC.may_optimize(interp::EscapeAnalyzer) = may_optimize(interp.native) +CC.may_compress(interp::EscapeAnalyzer) = may_compress(interp.native) +CC.may_discard_trees(interp::EscapeAnalyzer) = may_discard_trees(interp.native) +CC.verbose_stmt_info(interp::EscapeAnalyzer) = verbose_stmt_info(interp.native) + +CC.get_inference_cache(interp::EscapeAnalyzer) = get_inference_cache(interp.native) + +const GLOBAL_CODE_CACHE = IdDict{MethodInstance,CodeInstance}() +__clear_code_cache!() = empty!(GLOBAL_CODE_CACHE) + +function CC.code_cache(interp::EscapeAnalyzer) + worlds = WorldRange(get_world_counter(interp)) + return WorldView(GlobalCache(), worlds) +end + +struct GlobalCache end + +CC.haskey(wvc::WorldView{GlobalCache}, mi::MethodInstance) = haskey(GLOBAL_CODE_CACHE, mi) + +CC.get(wvc::WorldView{GlobalCache}, mi::MethodInstance, default) = get(GLOBAL_CODE_CACHE, mi, default) + +CC.getindex(wvc::WorldView{GlobalCache}, mi::MethodInstance) = getindex(GLOBAL_CODE_CACHE, mi) + +function CC.setindex!(wvc::WorldView{GlobalCache}, ci::CodeInstance, mi::MethodInstance) + GLOBAL_CODE_CACHE[mi] = ci + add_callback!(mi) # register the callback on invalidation + return nothing +end + +function add_callback!(linfo) + if !isdefined(linfo, :callbacks) + linfo.callbacks = Any[invalidate_cache!] + else + if !any(@nospecialize(cb)->cb===invalidate_cache!, linfo.callbacks) + push!(linfo.callbacks, invalidate_cache!) + end + end + return nothing +end + +function invalidate_cache!(replaced, max_world, depth = 0) + delete!(GLOBAL_CODE_CACHE, replaced) + + if isdefined(replaced, :backedges) + for mi in replaced.backedges + mi = mi::MethodInstance + if !haskey(GLOBAL_CODE_CACHE, mi) + continue # otherwise fall into infinite loop + end + invalidate_cache!(mi, max_world, depth+1) + end + end + return nothing +end + +function CC.optimize(interp::EscapeAnalyzer, opt::OptimizationState, params::OptimizationParams, @nospecialize(result)) + ir = run_passes_with_ea(interp, opt.src, opt) + return CC.finish(interp, opt, params, ir, result) +end + +# HACK enable copy and paste from Core.Compiler +function run_passes_with_ea end +register_init_hook!() do +@eval CC begin + function $(@__MODULE__).run_passes_with_ea(interp::$EscapeAnalyzer, ci::CodeInfo, sv::OptimizationState) + @timeit "convert" ir = convert_to_ircode(ci, sv) + @timeit "slot2reg" ir = slot2reg(ir, ci, sv) + # TODO: Domsorting can produce an updated domtree - no need to recompute here + @timeit "compact 1" ir = compact!(ir) + @timeit "Inlining" ir = ssa_inlining_pass!(ir, ir.linetable, sv.inlining, ci.propagate_inbounds) + # @timeit "verify 2" verify_ir(ir) + @timeit "compact 2" ir = compact!(ir) + nargs = let def = sv.linfo.def + isa(def, Method) ? Int(def.nargs) : 0 + end + @timeit "collect escape information" state = $find_escapes(ir, nargs) + cacheir = copy(ir) + # cache this result + $setindex!($GLOBAL_ESCAPE_CACHE, (state, cacheir), sv.linfo) + # return back the result + interp.ir = cacheir + interp.state = state + interp.linfo = sv.linfo + @timeit "SROA" ir = sroa_pass!(ir) + @timeit "ADCE" ir = adce_pass!(ir) + @timeit "type lift" ir = type_lift_pass!(ir) + @timeit "compact 3" ir = compact!(ir) + if JLOptions().debug_level == 2 + @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) + end + return ir + end +end +end # register_init_hook!() do + +# printing +# -------- + +import .CC: + widenconst, singleton_type +import .EA: + EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_FIELD_SETS, + ⊑, ⊏, __clear_escape_cache! + +# in order to run a whole analysis from ground zero (e.g. for benchmarking, etc.) +__clear_caches!() = (__clear_code_cache!(); __clear_escape_cache!()) + +function get_name_color(x::EscapeLattice, symbol::Bool = false) + getname(x) = string(nameof(x)) + if x == EA.NotAnalyzed() + name, color = (getname(EA.NotAnalyzed), "◌"), :plain + elseif EA.has_no_escape(x) + name, color = (getname(EA.NoEscape), "✓"), :green + elseif EA.NoEscape() ⊏ EA.ignore_fieldsets(x) ⊑ AllReturnEscape() + name, color = (getname(EA.ReturnEscape), "↑"), :cyan + elseif EA.NoEscape() ⊏ EA.ignore_fieldsets(x) ⊑ AllThrownEscape() + name, color = (getname(EA.ThrownEscape), "↓"), :yellow + elseif EA.has_all_escape(x) + name, color = (getname(EA.AllEscape), "X"), :red + else + name, color = (nothing, "*"), :red + end + name = symbol ? last(name) : first(name) + if name !== nothing && EA.has_fieldsets(x) + name = string(name, "′") + end + return name, color +end + +AllReturnEscape() = EscapeLattice(true, true, false, TOP_ESCAPE_SITES, BOT_FIELD_SETS) +AllThrownEscape() = EscapeLattice(true, false, true, TOP_ESCAPE_SITES, BOT_FIELD_SETS) + +# pcs = sprint(show, collect(x.EscapeSites); context=:limit=>true) +function Base.show(io::IO, x::EscapeLattice) + name, color = get_name_color(x) + if isnothing(name) + Base.@invoke show(io::IO, x::Any) + else + printstyled(io, name; color) + end +end +function Base.show(io::IO, ::MIME"application/prs.juno.inline", x::EscapeLattice) + name, color = get_name_color(x) + if isnothing(name) + return x # use fancy tree-view + else + printstyled(io, name; color) + end +end + +struct EscapeResult + ir::IRCode + state::EscapeState + linfo::Union{Nothing,MethodInstance} + EscapeResult(ir::IRCode, state::EscapeState, linfo::Union{Nothing,MethodInstance} = nothing) = + new(ir, state, linfo) +end +Base.show(io::IO, result::EscapeResult) = print_with_info(io, result.ir, result.state, result.linfo) +@eval Base.iterate(res::EscapeResult, state=1) = + return state > $(fieldcount(EscapeResult)) ? nothing : (getfield(res, state), state+1) + +# adapted from https://github.com/JuliaDebug/LoweredCodeUtils.jl/blob/4612349432447e868cf9285f647108f43bd0a11c/src/codeedges.jl#L881-L897 +function print_with_info(io::IO, + ir::IRCode, (; arguments, ssavalues)::EscapeState, linfo::Union{Nothing,MethodInstance}) + # print escape information on SSA values + function preprint(io::IO) + ft = ir.argtypes[1] + f = singleton_type(ft) + if f === nothing + f = widenconst(ft) + end + print(io, f, '(') + for (i, arg) in enumerate(arguments) + i == 1 && continue + c, color = get_name_color(arg, true) + printstyled(io, '_', i, "::", ir.argtypes[i], ' ', c; color) + i ≠ length(arguments) && print(io, ", ") + end + print(io, ')') + if !isnothing(linfo) + def = linfo.def + printstyled(io, " in ", (isa(def, Module) ? (def,) : (def.module, " at ", def.file, ':', def.line))...; color=:bold) + end + println(io) + end + + # print escape information on SSA values + # nd = ndigits(length(ssavalues)) + function preprint(io::IO, idx::Int) + c, color = get_name_color(ssavalues[idx], true) + # printstyled(io, lpad(idx, nd), ' ', c, ' '; color) + printstyled(io, rpad(c, 2), ' '; color) + end + + print_with_info(preprint, (args...)->nothing, io, ir) +end + +function print_with_info(preprint, postprint, io::IO, ir::IRCode) + io = IOContext(io, :displaysize=>displaysize(io)) + used = Base.IRShow.stmts_used(io, ir) + # line_info_preprinter = Base.IRShow.lineinfo_disabled + line_info_preprinter = function (io::IO, indent::String, idx::Int) + r = Base.IRShow.inline_linfo_printer(ir)(io, indent, idx) + idx ≠ 0 && preprint(io, idx) + return r + end + line_info_postprinter = Base.IRShow.default_expr_type_printer + preprint(io) + bb_idx_prev = bb_idx = 1 + for idx = 1:length(ir.stmts) + preprint(io, idx) + bb_idx = Base.IRShow.show_ir_stmt(io, ir, idx, line_info_preprinter, line_info_postprinter, used, ir.cfg, bb_idx) + postprint(io, idx, bb_idx != bb_idx_prev) + bb_idx_prev = bb_idx + end + max_bb_idx_size = ndigits(length(ir.cfg.blocks)) + line_info_preprinter(io, " "^(max_bb_idx_size + 2), 0) + postprint(io) + return nothing +end + +end # module EAUtils + +using .EAUtils: + analyze_escapes, + @analyze_escapes +export + analyze_escapes, + @analyze_escapes diff --git a/base/compiler/compiler.jl b/base/compiler/compiler.jl index 30054f51adeb6..72cbbff3ac213 100644 --- a/base/compiler/compiler.jl +++ b/base/compiler/compiler.jl @@ -130,7 +130,7 @@ include("compiler/stmtinfo.jl") include("compiler/abstractinterpretation.jl") include("compiler/typeinfer.jl") include("compiler/optimize.jl") # TODO: break this up further + extract utilities -include("compiler/EscapeAnalysis.jl") +include("compiler/EscapeAnalysis/EscapeAnalysis.jl") using .EscapeAnalysis include("compiler/bootstrap.jl") diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 9f8f9f8ae6c56..6ffd6cab2430a 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -448,13 +448,13 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) @timeit "SROA" ir = sroa_pass!(ir) @timeit "ADCE" ir = adce_pass!(ir) @timeit "type lift" ir = type_lift_pass!(ir) + @timeit "compact 3" ir = compact!(ir) nargs = let def = sv.linfo.def isa(def, Method) ? Int(def.nargs) : 0 end esc_state = find_escapes(ir, nargs) # setindex!(GLOBAL_ESCAPE_CACHE, esc_state, sv.linfo) @timeit "memory opt" ir = memory_opt!(ir, esc_state) - # @timeit "compact 3" ir = compact!(ir) if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) end From 2c466e12c2c3e3d0acb305a97585760088da6800 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sun, 26 Dec 2021 18:02:08 +0900 Subject: [PATCH 12/41] update to latest EA Adapted from . --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 399 ++++++++++-------- base/compiler/EscapeAnalysis/utils.jl | 20 +- base/compiler/optimize.jl | 6 +- base/compiler/ssair/passes.jl | 16 +- 4 files changed, 238 insertions(+), 203 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index de65c97341425..0486b07f2af11 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -2,6 +2,7 @@ baremodule EscapeAnalysis export find_escapes, + GLOBAL_ESCAPE_CACHE, has_not_analyzed, has_no_escape, has_return_escape, @@ -16,7 +17,7 @@ export const _TOP_MOD = ccall(:jl_base_relative_to, Any, (Any,), EscapeAnalysis)::Module # imports -import ._TOP_MOD: == +import ._TOP_MOD: ==, getindex, setindex! # usings import Core: MethodInstance, Const, Argument, SSAValue, PiNode, PhiNode, UpsilonNode, PhiCNode, @@ -58,7 +59,7 @@ else include(@__MODULE__, "compiler/EscapeAnalysis/disjoint_set.jl") end -const EscapeSet = IdSet{Any} +const EscapeSet = BitSet # XXX better to be IdSet{Int}? const EscapeSets = Vector{EscapeSet} """ @@ -318,53 +319,88 @@ end # TODO setup a more effient struct for cache # which can discard escape information on SSS values and arguments that don't join dispatch signature +const AliasSet = IntDisjointSet{Int} + """ - state::EscapeState + estate::EscapeState Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`: -- `state.arguments::Vector{EscapeLattice}`: escape information about "arguments"; +- `estate.arguments::Vector{EscapeLattice}`: escape information about "arguments"; note that "argument" can include both call arguments and slots appearing in analysis frame - `ssavalues::Vector{EscapeLattice}`: escape information about each SSA value - `aliaset::IntDisjointSet{Int}`: a disjoint set that maintains aliased arguments and SSA values """ struct EscapeState - arguments::Vector{EscapeLattice} - ssavalues::Vector{EscapeLattice} - aliasset::IntDisjointSet{Int} + escapes::Vector{EscapeLattice} + aliasset::AliasSet + nargs::Int +end +function EscapeState(nargs::Int, nstmts::Int) + escapes = EscapeLattice[ + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:(nargs+nstmts)] + aliaset = AliasSet(nargs+nstmts) + return EscapeState(escapes, aliaset, nargs) end -function EscapeState(nslots::Int, nargs::Int, nstmts::Int) - arguments = EscapeLattice[ - 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:nslots] - ssavalues = EscapeLattice[NotAnalyzed() for _ in 1:nstmts] - aliaset = AliasSet(nslots+nstmts) - return EscapeState(arguments, ssavalues, aliaset) +function getindex(estate::EscapeState, @nospecialize(x)) + if isa(x, Argument) || isa(x, SSAValue) + return estate.escapes[iridx(x, estate)] + else + return nothing + end +end +function setindex!(estate::EscapeState, v::EscapeLattice, @nospecialize(x)) + if isa(x, Argument) || isa(x, SSAValue) + estate.escapes[iridx(x, estate)] = v + end + return estate end -const AliasSet = IntDisjointSet{Int} -function alias_idx(@nospecialize(x), ir::IRCode) +""" + iridx(x, estate::EscapeState) -> xidx::Union{Int,Nothing} + +Tries to convert analyzable IR element `x::Union{Argument,SSAValue}` to +its unique identifier number `xidx` that is valid in the analysis context of `estate`. +Returns `nothing` if `x` isn't maintained by `estate` and thus unanalyzable (e.g. `x::GlobalRef`). + +`irval` can be used as an inverse function of `iridx`, i.e. +`irval(iridx(x::Union{Argument,SSAValue}, state), state) === x`. +""" +function iridx(@nospecialize(x), estate::EscapeState) if isa(x, Argument) - return x.n + xidx = x.n + @assert 1 ≤ xidx ≤ estate.nargs "invalid Argument" elseif isa(x, SSAValue) - return x.id + length(ir.argtypes) + xidx = x.id + estate.nargs else return nothing end + return xidx end -function alias_val(idx::Int, ir::IRCode) - n = length(ir.argtypes) - return idx > n ? SSAValue(idx-n) : Argument(idx) + +""" + irval(xidx::Int, estate::EscapeState) -> x::Union{Argument,SSAValue} + +Converts its unique identifier number `xidx` to the original IR element `x::Union{Argument,SSAValue}` +that is analyzable in the context of `estate`. + +`iridx` can be used as an inverse function of `irval`, i.e. +`iridx(irval(xidx, state), state) === xidx`. +""" +function irval(xidx::Int, estate::EscapeState) + x = xidx > estate.nargs ? SSAValue(xidx-estate.nargs) : Argument(xidx) + return x end -function get_aliases(aliasset::AliasSet, @nospecialize(key), ir::IRCode) - idx = alias_idx(key, ir) - idx === nothing && return nothing - root = find_root!(aliasset, idx) - if idx ≠ root || aliasset.ranks[idx] > 0 + +function getaliases(xidx::Int, estate::EscapeState) + aliasset = estate.aliasset + root = find_root!(aliasset, xidx) + if xidx ≠ root || aliasset.ranks[xidx] > 0 # the size of this alias set containing `key` is larger than 1, # collect the entire alias set - aliases = Union{Argument,SSAValue}[] - for i in 1:length(aliasset.parents) - if aliasset.parents[i] == root - push!(aliases, alias_val(i, ir)) + aliases = Int[] + for aidx in 1:length(aliasset.parents) + if aliasset.parents[aidx] == root + push!(aliases, aidx) end end return aliases @@ -373,14 +409,30 @@ function get_aliases(aliasset::AliasSet, @nospecialize(key), ir::IRCode) end end -# we preserve `IRCode` as well just for debugging purpose -const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Tuple{EscapeState,IRCode}}() +if _TOP_MOD !== Core.Compiler + struct EscapeCache + state::EscapeState + ir::IRCode # we preserve `IRCode` as well just for debugging purpose + end + const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,EscapeCache}() + argescapes_from_cache(cache::EscapeCache) = + cache.state.escapes[1:cache.state.nargs] +else + const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Vector{EscapeLattice}}() + argescapes_from_cache(cache::Vector{EscapeLattice}) = cache +end __clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) -const EscapeChange = Pair{Union{Argument,SSAValue},EscapeLattice} +const EscapeChange = Pair{Int,EscapeLattice} const AliasChange = Pair{Int,Int} const Changes = Vector{Union{EscapeChange,AliasChange}} +struct AnalysisState + ir::IRCode + estate::EscapeState + changes::Changes +end + """ find_escapes(ir::IRCode, nargs::Int) -> EscapeState @@ -388,12 +440,13 @@ Analyzes escape information in `ir`. `nargs` is the number of actual arguments of the analyzed call. """ function find_escapes(ir::IRCode, nargs::Int) - (; stmts, sptypes, argtypes) = ir + stmts = ir.stmts nstmts = length(stmts) # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties - state = EscapeState(length(argtypes), nargs, nstmts) + estate = EscapeState(nargs, nstmts) changes = Changes() # stashes changes that happen at current statement + astate = AnalysisState(ir, estate, changes) local debug_itr_counter = 0 while true @@ -410,30 +463,30 @@ function find_escapes(ir::IRCode, nargs::Int) if isa(stmt, Expr) head = stmt.head if head === :call - has_changes = escape_call!(ir, pc, stmt.args, state, changes) + has_changes = escape_call!(astate, pc, stmt.args) # TODO throwness ≠ "effect-free-ness" if !is_effect_free for x in stmt.args - add_escape_change!(x, ir, ThrownEscape(pc), changes) + add_escape_change!(astate, x, ThrownEscape(pc)) end else has_changes || continue end elseif head === :invoke - escape_invoke!(ir, pc, stmt.args, state, changes) + escape_invoke!(astate, pc, stmt.args) elseif head === :new || head === :splatnew - escape_new!(ir, pc, stmt.args, state, changes) + escape_new!(astate, pc, stmt.args) elseif head === :(=) lhs, rhs = stmt.args if isa(lhs, GlobalRef) # global store - add_escape_change!(rhs, ir, AllEscape(), changes) + add_escape_change!(astate, rhs, AllEscape()) else invalid_escape_assignment!(ir, pc) end elseif head === :foreigncall - escape_foreigncall!(ir, pc, stmt.args, state, changes) + escape_foreigncall!(astate, pc, stmt.args) elseif head === :throw_undef_if_not # XXX when is this expression inserted ? - add_escape_change!(stmt.args[1], ir, ThrownEscape(pc), changes) + add_escape_change!(astate, stmt.args[1], ThrownEscape(pc)) elseif is_meta_expr_head(head) # meta expressions doesn't account for any usages continue @@ -465,28 +518,28 @@ function find_escapes(ir::IRCode, nargs::Int) continue else for x in stmt.args - add_escape_change!(x, ir, AllEscape(), changes) + add_escape_change!(astate, x, AllEscape()) end end elseif isa(stmt, ReturnNode) if isdefined(stmt, :val) - add_escape_change!(stmt.val, ir, ReturnEscape(pc), changes) + add_escape_change!(astate, stmt.val, ReturnEscape(pc)) end elseif isa(stmt, PhiNode) - escape_edges!(ir, pc, stmt.values, state, changes) + escape_edges!(astate, pc, stmt.values) elseif isa(stmt, PiNode) - escape_val!(ir, pc, stmt, state, changes) + escape_val!(astate, pc, stmt) elseif isa(stmt, PhiCNode) - escape_edges!(ir, pc, stmt.values, state, changes) + escape_edges!(astate, pc, stmt.values) elseif isa(stmt, UpsilonNode) - escape_val!(ir, pc, stmt, state, changes) + escape_val!(astate, pc, stmt) elseif isa(stmt, GlobalRef) # global load - add_escape_change!(SSAValue(pc), ir, AllEscape(), changes) + add_escape_change!(astate, SSAValue(pc), AllEscape()) elseif isa(stmt, SSAValue) # NOTE after SROA, we may see SSA value as statement - info = state.ssavalues[pc] - add_escape_change!(stmt, ir, info, changes) - add_alias_change!(stmt, SSAValue(pc), ir, changes) + info = estate[SSAValue(pc)] + add_escape_change!(astate, stmt, info) + add_alias_change!(astate, stmt, SSAValue(pc)) else @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me continue @@ -494,7 +547,7 @@ function find_escapes(ir::IRCode, nargs::Int) isempty(changes) && continue - anyupdate |= propagate_changes!(state, changes, ir) + anyupdate |= propagate_changes!(estate, changes) empty!(changes) end @@ -508,95 +561,85 @@ function find_escapes(ir::IRCode, nargs::Int) # println("[EA] excessive iteration count found ", debug_itr_counter, " (", singleton_type(ir.argtypes[1]), ")") # end - return state + return estate end # propagate changes, and check convergence -function propagate_changes!(state::EscapeState, changes::Changes, ir::IRCode) +function propagate_changes!(estate::EscapeState, changes::Changes) local anychanged = false for change in changes if isa(change, EscapeChange) - anychanged |= propagate_escape_change!(state, change) - x, info = change - aliases = get_aliases(state.aliasset, x, ir) + anychanged |= propagate_escape_change!(estate, change) + xidx, info = change + aliases = getaliases(xidx, estate) if aliases !== nothing - for alias in aliases - morechange = EscapeChange(alias, info) - anychanged |= propagate_escape_change!(state, morechange) + for aidx in aliases + morechange = EscapeChange(aidx, info) + anychanged |= propagate_escape_change!(estate, morechange) end end else - anychanged |= propagate_alias_change!(state, change) + anychanged |= propagate_alias_change!(estate, change) end end return anychanged end -function propagate_escape_change!(state::EscapeState, change::EscapeChange) - x, info = change - if isa(x, Argument) - old = state.arguments[x.n] - new = old ⊔ info - if old ≠ new - state.arguments[x.n] = new - return true - end - else - x = x::SSAValue - old = state.ssavalues[x.id] - new = old ⊔ info - if old ≠ new - state.ssavalues[x.id] = new - return true - end +function propagate_escape_change!(estate::EscapeState, change::EscapeChange) + xidx, info = change + old = estate.escapes[xidx] + new = old ⊔ info + if old ≠ new + estate.escapes[xidx] = new + return true end return false end -function propagate_alias_change!(state::EscapeState, change::AliasChange) - x, y = change - xroot = find_root!(state.aliasset, x) - yroot = find_root!(state.aliasset, y) +function propagate_alias_change!(estate::EscapeState, change::AliasChange) + xidx, yidx = change + xroot = find_root!(estate.aliasset, xidx) + yroot = find_root!(estate.aliasset, yidx) if xroot ≠ yroot - union!(state.aliasset, xroot, yroot) + union!(estate.aliasset, xroot, yroot) return true end return false end -function add_escape_change!(@nospecialize(x), ir::IRCode, info::EscapeLattice, changes::Changes) - if isa(x, Argument) || isa(x, SSAValue) - if !isbitstype(widenconst(argextype(x, ir))) - push!(changes, EscapeChange(x, info)) +function add_escape_change!(astate::AnalysisState, @nospecialize(x), info::EscapeLattice) + xidx = iridx(x, astate.estate) + if xidx !== nothing + if !isbitstype(widenconst(argextype(x, astate.ir))) + push!(astate.changes, EscapeChange(xidx, info)) end end end -function add_alias_change!(@nospecialize(x), @nospecialize(y), ir::IRCode, changes::Changes) - xidx = alias_idx(x, ir) - yidx = alias_idx(y, ir) +function add_alias_change!(astate::AnalysisState, @nospecialize(x), @nospecialize(y)) + xidx = iridx(x, astate.estate) + yidx = iridx(y, astate.estate) if xidx !== nothing && yidx !== nothing - push!(changes, AliasChange(xidx, yidx)) + push!(astate.changes, AliasChange(xidx, yidx)) end end -function escape_edges!(ir::IRCode, pc::Int, edges::Vector{Any}, - state::EscapeState, changes::Changes) - info = state.ssavalues[pc] +function escape_edges!(astate::AnalysisState, pc::Int, edges::Vector{Any}) + info = astate.estate[SSAValue(pc)] for i in 1:length(edges) if isassigned(edges, i) v = edges[i] - add_escape_change!(v, ir, info, changes) - add_alias_change!(SSAValue(pc), v, ir, changes) + add_escape_change!(astate, v, info) + add_alias_change!(astate, SSAValue(pc), v) end end end -function escape_val!(ir::IRCode, pc::Int, x, state::EscapeState, changes::Changes) +function escape_val!(astate::AnalysisState, pc::Int, x) if isdefined(x, :val) - info = state.ssavalues[pc] - add_escape_change!(x.val, ir, info, changes) - add_alias_change!(SSAValue(pc), x.val, ir, changes) + info = astate.estate[SSAValue(pc)] + add_escape_change!(astate, x.val, info) + add_alias_change!(astate, SSAValue(pc), x.val) end end @@ -604,42 +647,39 @@ end # soemthing like below: it essentially incorporates forward escape propagation in our default # backward propagation, and leads to inefficient convergence that requires more iterations # # lhs = rhs: propagate escape information of `rhs` to `lhs` -# function escape_alias!(@nospecialize(lhs), @nospecialize(rhs), -# ir::IRCode, state::EscapeState, changes::Changes) -# if isa(rhs, SSAValue) -# vinfo = state.ssavalues[rhs.id] -# elseif isa(rhs, Argument) -# vinfo = state.arguments[rhs.n] +# function escape_alias!(astate::AnalysisState, @nospecialize(lhs), @nospecialize(rhs)) +# if isa(rhs, SSAValue) || isa(rhs, Argument) +# vinfo = astate.estate[rhs] # else # return # end -# add_escape_change!(lhs, ir, vinfo, changes) +# add_escape_change!(astate, lhs, vinfo) # end -function escape_invoke!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) +function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) linfo = first(args)::MethodInstance cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) args = args[2:end] if cache === nothing for x in args - add_escape_change!(x, ir, AllEscape(), changes) + add_escape_change!(astate, x, AllEscape()) end else - (linfostate, #=, ir::IRCode=#) = cache - retinfo = state.ssavalues[pc] # escape information imposed on the call statement + argescapes = argescapes_from_cache(cache) + retinfo = astate.estate[SSAValue(pc)] # escape information imposed on the call statement method = linfo.def::Method nargs = Int(method.nargs) for i in 1:length(args) arg = args[i] if i ≤ nargs - arginfo = linfostate.arguments[i] + argi = i else # handle isva signature: COMBAK will this be invalid once we take alias information into account ? - arginfo = linfostate.arguments[nargs] + argi = nargs end - isempty(arginfo.ReturnEscape) && invalid_escape_invoke!(ir, linfo) + arginfo = argescapes[argi] + isempty(arginfo.ReturnEscape) && invalid_escape_invoke!(astate, linfo, linfo_estate) info = from_interprocedural(arginfo, retinfo, pc) - add_escape_change!(arg, ir, info, changes) + add_escape_change!(astate, arg, info) end end end @@ -672,9 +712,9 @@ function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc end end -@noinline function invalid_escape_invoke!(ir::IRCode, linfo::MethodInstance) - @eval Main (ir = $ir; linfo = $linfo) - error("invalid escape lattice element returned from inter-procedural context: inspect `Main.ir` and `Main.linfo`") +@noinline function invalid_escape_invoke!(astate::AnalysisState, linfo::MethodInstance, linfo_estate::EscapeState) + @eval Main (astate = $astate; linfo = $linfo; linfo_estate = $linfo_estate) + error("invalid escape lattice element returned from inter-procedural context: inspect `Main.astate`, `Main.linfo` and `Main.linfo_estate`") end @noinline function invalid_escape_assignment!(ir::IRCode, pc::Int) @@ -682,9 +722,8 @@ end error("unexpected assignment found: inspect `Main.pc` and `Main.pc`") end -function escape_new!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) - objinfo = state.ssavalues[pc] +function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) + objinfo = astate.estate[SSAValue(pc)] if objinfo == NotAnalyzed() objinfo = NoEscape() end @@ -694,43 +733,38 @@ function escape_new!(ir::IRCode, pc::Int, args::Vector{Any}, # the fields couldn't be analyzed precisely: directly propagate the escape information # of this object to all its fields (which is the most conservative option) for i in 2:nargs - add_escape_change!(args[i], ir, objinfo, changes) + add_escape_change!(astate, args[i], objinfo) end else # fields are known: propagate escape information imposed on recorded possibilities nf = length(FieldEscapes) for i in 2:nargs i-1 > nf && break # may happen when e.g. ϕ-node merges values with different types - escape_field!(args[i], FieldEscapes[i-1], ir, state, changes) + escape_field!(astate, args[i], FieldEscapes[i-1]) end end end -function escape_field!(@nospecialize(v), FieldEscape::EscapeSet, ir::IRCode, state::EscapeState, changes::Changes) - for x in FieldEscape - if isa(x, SSAValue) - add_escape_change!(v, ir, state.ssavalues[x.id], changes) - elseif isa(x, Argument) - add_escape_change!(v, ir, state.arguments[x.n], changes) - else - continue - end - add_alias_change!(v, x, ir, changes) +function escape_field!(astate::AnalysisState, @nospecialize(v), FieldEscape::EscapeSet) + estate = astate.estate + for xidx in FieldEscape + x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape + add_escape_change!(astate, v, estate[x]) + add_alias_change!(astate, v, x) end end # escape every argument `(args[6:length(args[3])])` and the name `args[1]` # TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls -function escape_foreigncall!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) +function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) foreigncall_nargs = length((args[3])::SimpleVector) name = args[1] # if normalize(name) === :jl_gc_add_finalizer_th # # add `FinalizerEscape` ? # end - add_escape_change!(name, ir, ThrownEscape(pc), changes) + add_escape_change!(astate, name, ThrownEscape(pc)) for i in 6:5+foreigncall_nargs - add_escape_change!(args[i], ir, ThrownEscape(pc), changes) + add_escape_change!(astate, args[i], ThrownEscape(pc)) end end @@ -738,21 +772,21 @@ end # TODO implement more builtins, make them more accurate # TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? -function escape_call!(ir::IRCode, pc::Int, args::Vector{Any}, - state::EscapeState, changes::Changes) +function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) + ir = astate.ir ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) f = singleton_type(ft) if isa(f, Core.IntrinsicFunction) return false # COMBAK we may break soundness here, e.g. `pointerref` end - result = escape_builtin!(f, ir, pc, args, state, changes) + result = escape_builtin!(f, astate, pc, args) if result === false return false # nothing to propagate elseif result === missing # if this call hasn't been handled by any of pre-defined handlers, # we escape this call conservatively for i in 2:length(args) - add_escape_change!(args[i], ir, AllEscape(), changes) + add_escape_change!(astate, args[i], AllEscape()) end return true else @@ -771,54 +805,54 @@ escape_builtin!(::typeof(===), _...) = return false escape_builtin!(::typeof(isdefined), _...) = return false escape_builtin!(::typeof(throw), _...) = return false -function escape_builtin!(::typeof(Core.ifelse), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) +function escape_builtin!(::typeof(Core.ifelse), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) == 4 || return nothing f, cond, th, el = args - info = state.ssavalues[pc] - condt = argextype(cond, ir) ret = SSAValue(pc) + info = astate.estate[ret] + condt = argextype(cond, astate.ir) if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) if cond - add_escape_change!(th, ir, info, changes) - add_alias_change!(th, ret, ir, changes) + add_escape_change!(astate, th, info) + add_alias_change!(astate, th, ret) else - add_escape_change!(el, ir, info, changes) - add_alias_change!(el, ret, ir, changes) + add_escape_change!(astate, el, info) + add_alias_change!(astate, el, ret) end else - add_escape_change!(th, ir, info, changes) - add_escape_change!(el, ir, info, changes) - add_alias_change!(th, ret, ir, changes) - add_alias_change!(el, ret, ir, changes) + add_escape_change!(astate, th, info) + add_alias_change!(astate, th, ret) + add_escape_change!(astate, el, info) + add_alias_change!(astate, el, ret) end return nothing end -function escape_builtin!(::typeof(typeassert), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) +function escape_builtin!(::typeof(typeassert), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) == 3 || return nothing f, obj, typ = args - info = state.ssavalues[pc] - add_escape_change!(obj, ir, info, changes) - add_alias_change!(SSAValue(pc), obj, ir, changes) + ret = SSAValue(pc) + info = astate.estate[ret] + add_escape_change!(astate, obj, info) + add_alias_change!(astate, ret, obj) return nothing end -function escape_builtin!(::typeof(tuple), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) - escape_new!(ir, pc, args, state, changes) +function escape_builtin!(::typeof(tuple), astate::AnalysisState, pc::Int, args::Vector{Any}) + escape_new!(astate, pc, args) return nothing end -function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) +function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 3 || return nothing + ir, estate = astate.ir, astate.estate obj = args[2] typ = widenconst(argextype(obj, ir)) if hasintersect(typ, Module) # global load - add_escape_change!(SSAValue(pc), ir, AllEscape(), changes) + add_escape_change!(astate, SSAValue(pc), AllEscape()) end - if isa(obj, SSAValue) - objinfo = state.ssavalues[obj.id] - elseif isa(obj, Argument) - objinfo = state.arguments[obj.n] + if isa(obj, SSAValue) || isa(obj, Argument) + objinfo = estate[obj] else return end @@ -835,11 +869,11 @@ function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{A # the field couldn't be analyzed precisely: directly propagate the escape information # imposed on the return value of this `getfield` call to the object (which is the most conservative option) # but also with updated field information - ssainfo = state.ssavalues[pc] + ssainfo = estate[SSAValue(pc)] if ssainfo == NotAnalyzed() ssainfo = NoEscape() end - add_escape_change!(obj, ir, EscapeLattice(ssainfo, TOP_FIELD_SETS), changes) + add_escape_change!(astate, obj, EscapeLattice(ssainfo, TOP_FIELD_SETS)) else # fields are known: record the return value of this `getfield` call as a possibility that imposes escape FieldEscapes = copy(FieldEscapes) @@ -853,28 +887,27 @@ function escape_builtin!(::typeof(getfield), ir::IRCode, pc::Int, args::Vector{A end if fidx !== nothing # the field is known precisely: propagate this escape information to the field - push!(FieldEscapes[fidx], SSAValue(pc)) + push!(FieldEscapes[fidx], iridx(SSAValue(pc), estate)) else # the field isn't known precisely: propagate this escape information to all the fields for FieldEscape in FieldEscapes - push!(FieldEscape, SSAValue(pc)) + push!(FieldEscape, iridx(SSAValue(pc), estate)) end end - add_escape_change!(obj, ir, EscapeLattice(objinfo, FieldEscapes), changes) + add_escape_change!(astate, obj, EscapeLattice(objinfo, FieldEscapes)) end return nothing end -function escape_builtin!(::typeof(setfield!), ir::IRCode, pc::Int, args::Vector{Any}, state::EscapeState, changes::Changes) +function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 4 || return nothing + ir, estate = astate.ir, astate.estate obj, fld, val = args[2:4] - if isa(obj, SSAValue) - objinfo = state.ssavalues[obj.id] - elseif isa(obj, Argument) - objinfo = state.arguments[obj.n] + if isa(obj, SSAValue) || isa(obj, Argument) + objinfo = estate[obj] else # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively - add_escape_change!(val, ir, AllEscape(), changes) + add_escape_change!(astate, val, AllEscape()) return end FieldEscapes = objinfo.FieldEscapes @@ -887,16 +920,16 @@ function escape_builtin!(::typeof(setfield!), ir::IRCode, pc::Int, args::Vector{ # unsuccessful field analysis: update obj's escape information with new field information FieldEscapes = EscapeSet[EscapeSet() for _ in 1:nfields] objinfo = EscapeLattice(objinfo, FieldEscapes) - add_escape_change!(obj, ir, objinfo, changes) + add_escape_change!(astate, obj, objinfo) @goto add_field_escape end # unsuccessful field analysis: update obj's escape information with new field information objinfo = EscapeLattice(objinfo, TOP_FIELD_SETS) - add_escape_change!(obj, ir, objinfo, changes) + add_escape_change!(astate, obj, objinfo) end # the field couldn't be analyzed precisely: directly propagate the escape information # of this object to the field (which is the most conservative option) - add_escape_change!(val, ir, objinfo, changes) + add_escape_change!(astate, val, objinfo) else # fields are known: propagate escape information imposed on recorded possibilities typ = widenconst(argextype(obj, ir)) @@ -909,20 +942,20 @@ function escape_builtin!(::typeof(setfield!), ir::IRCode, pc::Int, args::Vector{ end if fidx !== nothing # the field is known precisely: propagate this escape information to the field - escape_field!(val, FieldEscapes[fidx], ir, state, changes) + escape_field!(astate, val, FieldEscapes[fidx]) else # the field isn't known precisely: propagate this escape information to all the fields for FieldEscape in FieldEscapes - escape_field!(val, FieldEscape, ir, state, changes) + escape_field!(astate, val, FieldEscape) end end end # also propagate escape information imposed on the return value of this `setfield!` - ssainfo = state.ssavalues[pc] + ssainfo = estate[SSAValue(pc)] if ssainfo == NotAnalyzed() ssainfo = NoEscape() end - add_escape_change!(val, ir, ssainfo, changes) + add_escape_change!(astate, val, ssainfo) return nothing end diff --git a/base/compiler/EscapeAnalysis/utils.jl b/base/compiler/EscapeAnalysis/utils.jl index a792e9fb9b4d9..eba5f076f914b 100644 --- a/base/compiler/EscapeAnalysis/utils.jl +++ b/base/compiler/EscapeAnalysis/utils.jl @@ -61,7 +61,7 @@ import Core: import .CC: OptimizationState, IRCode import .EA: - find_escapes, GLOBAL_ESCAPE_CACHE + find_escapes, GLOBAL_ESCAPE_CACHE, EscapeCache mutable struct EscapeAnalyzer{State} <: AbstractInterpreter native::NativeInterpreter @@ -158,7 +158,7 @@ register_init_hook!() do @timeit "collect escape information" state = $find_escapes(ir, nargs) cacheir = copy(ir) # cache this result - $setindex!($GLOBAL_ESCAPE_CACHE, (state, cacheir), sv.linfo) + $setindex!($GLOBAL_ESCAPE_CACHE, $EscapeCache(state, cacheir), sv.linfo) # return back the result interp.ir = cacheir interp.state = state @@ -178,11 +178,10 @@ end # register_init_hook!() do # printing # -------- -import .CC: - widenconst, singleton_type +import Core: Argument, SSAValue +import .CC: widenconst, singleton_type import .EA: - EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_FIELD_SETS, - ⊑, ⊏, __clear_escape_cache! + EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_FIELD_SETS, ⊑, ⊏, __clear_escape_cache! # in order to run a whole analysis from ground zero (e.g. for benchmarking, etc.) __clear_caches!() = (__clear_code_cache!(); __clear_escape_cache!()) @@ -243,7 +242,7 @@ Base.show(io::IO, result::EscapeResult) = print_with_info(io, result.ir, result. # adapted from https://github.com/JuliaDebug/LoweredCodeUtils.jl/blob/4612349432447e868cf9285f647108f43bd0a11c/src/codeedges.jl#L881-L897 function print_with_info(io::IO, - ir::IRCode, (; arguments, ssavalues)::EscapeState, linfo::Union{Nothing,MethodInstance}) + ir::IRCode, state::EscapeState, linfo::Union{Nothing,MethodInstance}) # print escape information on SSA values function preprint(io::IO) ft = ir.argtypes[1] @@ -252,11 +251,12 @@ function print_with_info(io::IO, f = widenconst(ft) end print(io, f, '(') - for (i, arg) in enumerate(arguments) + for i in 1:state.nargs + arg = state[Argument(i)] i == 1 && continue c, color = get_name_color(arg, true) printstyled(io, '_', i, "::", ir.argtypes[i], ' ', c; color) - i ≠ length(arguments) && print(io, ", ") + i ≠ state.nargs && print(io, ", ") end print(io, ')') if !isnothing(linfo) @@ -269,7 +269,7 @@ function print_with_info(io::IO, # print escape information on SSA values # nd = ndigits(length(ssavalues)) function preprint(io::IO, idx::Int) - c, color = get_name_color(ssavalues[idx], true) + c, color = get_name_color(state[SSAValue(idx)], true) # printstyled(io, lpad(idx, nd), ' ', c, ' '; color) printstyled(io, rpad(c, 2), ' '; color) end diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 6ffd6cab2430a..d83dfac671379 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -452,9 +452,9 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) nargs = let def = sv.linfo.def isa(def, Method) ? Int(def.nargs) : 0 end - esc_state = find_escapes(ir, nargs) - # setindex!(GLOBAL_ESCAPE_CACHE, esc_state, sv.linfo) - @timeit "memory opt" ir = memory_opt!(ir, esc_state) + estate = find_escapes(ir, nargs) + setindex!(GLOBAL_ESCAPE_CACHE, estate.escapes[1:estate.nargs], sv.linfo) + @timeit "memory opt" ir = memory_opt!(ir, estate) if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) end diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 87da18716369b..0742cafe032c0 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1407,7 +1407,8 @@ function _is_known_fcall(stmt::Expr, funcs) return false end -function memory_opt!(ir::IRCode, escape_state) +function memory_opt!(ir::IRCode, estate) + estate = estate::EscapeAnalysis.EscapeState compact = IncrementalCompact(ir, false) # relevant = IdSet{Int}() # allocations revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in @@ -1434,22 +1435,23 @@ function memory_opt!(ir::IRCode, escape_state) push!(maybecopies, idx) # elseif is_array_allocation(stmt) # push!(relevant, idx) - elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue) - push!(revisit, idx) + elseif is_known_call(stmt, Core.arrayfreeze, compact) + if isa(stmt.args[2], SSAValue) + push!(revisit, idx) + end end end ir = finish(compact) isempty(revisit) && isempty(maybecopies) && return ir - domtree = construct_domtree(ir.cfg.blocks) + # domtree = construct_domtree(ir.cfg.blocks) for idx in revisit stmt = ir.stmts[idx][:inst]::Expr - id = stmt.args[2].id + arg = stmt.args[2]::SSAValue # if our escape analysis has determined that this array doesn't escape, we can potentially eliminate an allocation - # @eval Main (ir = $ir; rev = $revisit; esc_state = $escape_state) - has_no_escape(escape_state.ssavalues[id]) || continue + has_no_escape(estate[arg]) || continue # # We're ok to steal the memory if we don't dominate any uses # ok = true From 235ab8443bedf69937d860e7464f90b6f8778538 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Tue, 28 Dec 2021 10:18:15 -0800 Subject: [PATCH 13/41] Restrict arrayfreeze/thaw arg # in base/compiler/tfuncs.jl Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> --- base/compiler/tfuncs.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index ab14cce83fe11..0f8ee3d8b8ff1 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1569,7 +1569,6 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp return tuple_tfunc(argtypes) elseif f === Core.arrayfreeze || f === Core.arraythaw if length(argtypes) != 1 - isva && return Any return Bottom end a = widenconst(argtypes[1]) From 0e5833bea0a6cd40eb3d81a3b840564c7ee85e8d Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Tue, 28 Dec 2021 13:20:41 -0500 Subject: [PATCH 14/41] Restrict nargs of arrayfreeze/thaw to 1 in builtins.c --- src/builtins.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 9769195b353a0..abd42b23daec7 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1407,7 +1407,7 @@ JL_CALLABLE(jl_f_maybecopy) // instead, calls to it are analyzed and replaced with either a call to copy // or directly replaced with the object itself that is the target of the maybecopy // therefore, we just check that there is one argument and do a no-op - JL_NARGSV(maybecopy, 1); + JL_NARGS(maybecopy, 1, 1); JL_TYPECHK(maybecopy, array, args[0]); jl_array_t *a = (jl_array_t*)args[0]; jl_array_t *na = jl_array_copy(a); @@ -1674,7 +1674,7 @@ JL_CALLABLE(jl_f__equiv_typedef) JL_CALLABLE(jl_f_arrayfreeze) { - JL_NARGSV(arrayfreeze, 1); + JL_NARGS(arrayfreeze, 1, 1); JL_TYPECHK(arrayfreeze, array, args[0]); jl_array_t *a = (jl_array_t*)args[0]; jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, @@ -1693,7 +1693,7 @@ JL_CALLABLE(jl_f_mutating_arrayfreeze) // N.B.: These error checks pretend to be arrayfreeze since this is a drop // in replacement and we don't want to change the visible error type in the // optimizer - JL_NARGSV(arrayfreeze, 1); + JL_NARGS(arrayfreeze, 1, 1); JL_TYPECHK(arrayfreeze, array, args[0]); jl_array_t *a = (jl_array_t*)args[0]; jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, @@ -1704,7 +1704,7 @@ JL_CALLABLE(jl_f_mutating_arrayfreeze) JL_CALLABLE(jl_f_arraythaw) { - JL_NARGSV(arraythaw, 1); + JL_NARGS(arraythaw, 1, 1); if (((jl_datatype_t*)jl_typeof(args[0]))->name != jl_immutable_array_typename) { jl_type_error("arraythaw", (jl_value_t*)jl_immutable_array_type, args[0]); } From de437d430f13a0f1f9019566b329668c1869c0db Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 4 Jan 2022 16:37:01 +0900 Subject: [PATCH 15/41] update EA and get array supports --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 744 ++++++++++++++---- base/compiler/EscapeAnalysis/utils.jl | 12 +- 2 files changed, 578 insertions(+), 178 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 0486b07f2af11..605b139ad264a 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -21,16 +21,17 @@ import ._TOP_MOD: ==, getindex, setindex! # usings import Core: MethodInstance, Const, Argument, SSAValue, PiNode, PhiNode, UpsilonNode, PhiCNode, - ReturnNode, GotoNode, GotoIfNot, SimpleVector + ReturnNode, GotoNode, GotoIfNot, SimpleVector, sizeof, ifelse, arrayset, arrayref, + arraysize import ._TOP_MOD: # Base definitions @__MODULE__, @eval, @assert, @nospecialize, @inbounds, @inline, @noinline, @label, @goto, - !, !==, !=, ≠, +, -, ≤, <, ≥, >, &, |, include, error, missing, + !, !==, !=, ≠, +, -, ≤, <, ≥, >, &, |, include, error, missing, copy, Vector, BitSet, IdDict, IdSet, ∪, ⊆, ∩, :, length, get, first, last, in, isempty, - isassigned, push!, empty!, max, min + isassigned, push!, empty!, max, min, Csize_t import Core.Compiler: # Core.Compiler specific definitions - isbitstype, isexpr, is_meta_expr_head, copy, println, + isbitstype, isexpr, is_meta_expr_head, println, IRCode, IR_FLAG_EFFECT_FREE, widenconst, argextype, singleton_type, fieldcount_noerror, - try_compute_fieldidx, hasintersect + try_compute_fieldidx, hasintersect, ⊑ as ⊑ₜ, intrinsic_nothrow if isdefined(Core.Compiler, :try_compute_field) import Core.Compiler: try_compute_field @@ -53,14 +54,44 @@ else end end +if isdefined(Core.Compiler, :array_builtin_common_typecheck) && + isdefined(Core.Compiler, :arrayset_typecheck) + import Core.Compiler: array_builtin_common_typecheck, arrayset_typecheck +else + function array_builtin_common_typecheck( + @nospecialize(boundcheck), @nospecialize(ary), + argtypes::Vector{Any}, first_idx_idx::Int) + (boundcheck ⊑ₜ Bool && ary ⊑ₜ Array) || return false + for i = first_idx_idx:length(argtypes) + argtypes[i] ⊑ₜ Int || return false + end + return true + end + function arrayset_typecheck(@nospecialize(atype), @nospecialize(elm)) + # Check that we can determine the element type + atype = widenconst(atype) + isa(atype, DataType) || return false + ap1 = atype.parameters[1] + isa(ap1, Type) || return false + # Check that the element type is compatible with the element we're assigning + elm ⊑ₜ ap1 || return false + return true + end +end + if _TOP_MOD !== Core.Compiler include(@__MODULE__, "disjoint_set.jl") else include(@__MODULE__, "compiler/EscapeAnalysis/disjoint_set.jl") end -const EscapeSet = BitSet # XXX better to be IdSet{Int}? -const EscapeSets = Vector{EscapeSet} +# XXX better to be IdSet{Int}? +const FieldEscape = BitSet +const FieldEscapes = Vector{BitSet} +# for x in ArrayEscapes: +# - x::Int: `irval(x, estate)` imposes escapes on the array elements +# - x::SSAValue: SSA statement (x.id) can potentially escape array elements via BoundsError +const ArrayEscapes = IdSet{Union{Int,SSAValue}} """ x::EscapeLattice @@ -73,14 +104,17 @@ A lattice for escape information, which holds the following properties: - `x.ThrownEscape::Bool`: indicates `x` may escape to somewhere through an exception - `x.EscapeSites::BitSet`: records SSA statements where `x` can escape via any of `ReturnEscape` or `ThrownEscape` -- `x.FieldEscapes::Union{Vector{IdSet{Any}},Bool}`: maintains all possible values that impose - escape information on fields of `x`: - * `x.FieldEscapes === false` indicates the fields of `x` isn't analyzed yet - * `x.FieldEscapes === true` indicates the fields of `x` can't be analyzed, e.g. the type of `x` - is not known or is not concrete and thus its fields can't be known precisely - * otherwise `x.FieldEscapes::Vector{IdSet{Any}}` holds all the possible values that can escape - fields of `x`, which allows EA to propagate propagate escape information imposed on a field +- `x.AliasEscapes::Union{FieldEscapes,ArrayEscapes,Bool}`: maintains all possible values + that may escape objects that can be referenced from `x`: + * `x.AliasEscapes === false` indicates the fields/elements of `x` isn't analyzed yet + * `x.AliasEscapes === true` indicates the fields/elements of `x` can't be analyzed, + e.g. the type of `x` is not known or is not concrete and thus its fields/elements + can't be known precisely + * `x.AliasEscapes::FieldEscapes` records all the possible values that can escape fields of `x`, + which allows EA to propagate propagate escape information imposed on a field of `x` to its values (by analyzing `Expr(:new, ...)` and `setfield!(x, ...)`). + * `x.AliasEscapes::ArrayEscapes` records all the possible values that can escape elements of `x`, + or all SSA staements that can potentially escape elements of `x` via `BoundsError`. - `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through `setfield!` on argument(s) * `-1` : no escape @@ -104,40 +138,42 @@ struct EscapeLattice ReturnEscape::Bool ThrownEscape::Bool EscapeSites::BitSet - FieldEscapes::Union{EscapeSets,Bool} + AliasEscapes #::Union{FieldEscapes,ArrayEscapes,Bool} # TODO: ArgEscape::Int - function EscapeLattice(Analyzed::Bool, - ReturnEscape::Bool, - ThrownEscape::Bool, - EscapeSites::BitSet, - FieldEscapes::Union{EscapeSets,Bool}, - ) - @nospecialize FieldEscapes + function EscapeLattice( + Analyzed::Bool, + ReturnEscape::Bool, + ThrownEscape::Bool, + EscapeSites::BitSet, + AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=#, + ) + @nospecialize AliasEscapes return new( Analyzed, ReturnEscape, ThrownEscape, EscapeSites, - FieldEscapes, + AliasEscapes, ) end - function EscapeLattice(x::EscapeLattice, - # non-concrete fields should be passed as default arguments - # in order to avoid allocating non-concrete `NamedTuple`s - FieldEscapes::Union{EscapeSets,Bool} = x.FieldEscapes; - Analyzed::Bool = x.Analyzed, - ReturnEscape::Bool = x.ReturnEscape, - ThrownEscape::Bool = x.ThrownEscape, - EscapeSites::BitSet = x.EscapeSites, - ) - @nospecialize FieldEscapes + function EscapeLattice( + x::EscapeLattice, + # non-concrete fields should be passed as default arguments + # in order to avoid allocating non-concrete `NamedTuple`s + AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=# = x.AliasEscapes; + Analyzed::Bool = x.Analyzed, + ReturnEscape::Bool = x.ReturnEscape, + ThrownEscape::Bool = x.ThrownEscape, + EscapeSites::BitSet = x.EscapeSites, + ) + @nospecialize AliasEscapes return new( Analyzed, ReturnEscape, ThrownEscape, EscapeSites, - FieldEscapes, + AliasEscapes, ) end end @@ -147,28 +183,28 @@ const BOT_ESCAPE_SITES = BitSet() const ARGUMENT_ESCAPE_SITES = BitSet(0) const TOP_ESCAPE_SITES = BitSet(0:100_000) -const BOT_FIELD_SETS = false -const TOP_FIELD_SETS = true +const BOT_ALIAS_ESCAPES = false +const TOP_ALIAS_ESCAPES = true # the constructors -NotAnalyzed() = EscapeLattice(false, false, false, BOT_ESCAPE_SITES, BOT_FIELD_SETS) # not formally part of the lattice -NoEscape() = EscapeLattice(true, false, false, BOT_ESCAPE_SITES, BOT_FIELD_SETS) -ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc), BOT_FIELD_SETS) -ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc), BOT_FIELD_SETS) -ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES, TOP_FIELD_SETS) # TODO allow interprocedural field analysis? -AllEscape() = EscapeLattice(true, true, true, TOP_ESCAPE_SITES, TOP_FIELD_SETS) +NotAnalyzed() = EscapeLattice(false, false, false, BOT_ESCAPE_SITES, BOT_ALIAS_ESCAPES) # not formally part of the lattice +NoEscape() = EscapeLattice(true, false, false, BOT_ESCAPE_SITES, BOT_ALIAS_ESCAPES) +ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc), BOT_ALIAS_ESCAPES) +ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc), BOT_ALIAS_ESCAPES) +ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES, TOP_ALIAS_ESCAPES) # TODO allow interprocedural field analysis? +AllEscape() = EscapeLattice(true, true, true, TOP_ESCAPE_SITES, TOP_ALIAS_ESCAPES) # Convenience names for some ⊑ queries has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() -has_no_escape(x::EscapeLattice) = ignore_fieldsets(x) ⊑ NoEscape() +has_no_escape(x::EscapeLattice) = ignore_aliasescapes(x) ⊑ NoEscape() has_return_escape(x::EscapeLattice) = x.ReturnEscape has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites has_thrown_escape(x::EscapeLattice) = x.ThrownEscape has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x -ignore_fieldsets(x::EscapeLattice) = EscapeLattice(x, BOT_FIELD_SETS) -has_fieldsets(x::EscapeLattice) = !isa(x.FieldEscapes, Bool) +ignore_aliasescapes(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_ESCAPES) +has_aliasescapes(x::EscapeLattice) = !isa(x.AliasEscapes, Bool) # TODO is_sroa_eligible: consider throwness? @@ -177,7 +213,13 @@ has_fieldsets(x::EscapeLattice) = !isa(x.FieldEscapes, Bool) Queries allocation eliminability by SROA. """ -is_sroa_eligible(x::EscapeLattice) = x.FieldEscapes !== TOP_FIELD_SETS && !has_return_escape(x) +function is_sroa_eligible(x::EscapeLattice) + if x.AliasEscapes === false || # allows this query to work for immutables since we don't impose escape on them + isa(x.AliasEscapes, FieldEscapes) + return !has_return_escape(x) # TODO technically we also need to check !has_thrown_escape(x) as well + end + return false +end """ can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool @@ -193,12 +235,15 @@ can_elide_finalizer(x::EscapeLattice, pc::Int) = # we need to make sure this `==` operator corresponds to lattice equality rather than object equality, # otherwise `propagate_changes` can't detect the convergence x::EscapeLattice == y::EscapeLattice = begin - xf, yf = x.FieldEscapes, y.FieldEscapes + xf, yf = x.AliasEscapes, y.AliasEscapes if isa(xf, Bool) - isa(yf, Bool) || return false xf === yf || return false + elseif isa(xf, FieldEscapes) + isa(yf, FieldEscapes) || return false + xf == yf || return false else - isa(yf, Bool) && return false + xf = xf::ArrayEscapes + isa(yf, ArrayEscapes) || return false xf == yf || return false end return x.Analyzed === y.Analyzed && @@ -214,19 +259,25 @@ end The non-strict partial order over `EscapeLattice`. """ x::EscapeLattice ⊑ y::EscapeLattice = begin - xf, yf = x.FieldEscapes, y.FieldEscapes + xf, yf = x.AliasEscapes, y.AliasEscapes if isa(xf, Bool) xf && yf !== true && return false - else - if isa(yf, Bool) - yf === false && return false - else - xf, yf = xf::EscapeSets, yf::EscapeSets + elseif isa(xf, FieldEscapes) + if isa(yf, FieldEscapes) xn, yn = length(xf), length(yf) xn > yn && return false for i in 1:xn xf[i] ⊆ yf[i] || return false end + else + yf === true || return false + end + else + xf = xf::ArrayEscapes + if isa(yf, ArrayEscapes) + xf ⊆ yf || return false + else + yf === true || return false end end if x.Analyzed ≤ y.Analyzed && @@ -261,24 +312,34 @@ x::EscapeLattice ⋤ y::EscapeLattice = !(y ⊑ x) Computes the join of `x` and `y` in the partial order defined by `EscapeLattice`. """ x::EscapeLattice ⊔ y::EscapeLattice = begin - xf, yf = x.FieldEscapes, y.FieldEscapes + xf, yf = x.AliasEscapes, y.AliasEscapes if xf === true || yf === true - FieldEscapes = true + AliasEscapes = true elseif xf === false - FieldEscapes = yf + AliasEscapes = yf elseif yf === false - FieldEscapes = xf - else - xf, yf = xf::EscapeSets, yf::EscapeSets - xn, yn = length(xf), length(yf) - nmax, nmin = max(xn, yn), min(xn, yn) - FieldEscapes = EscapeSets(undef, nmax) - for i in 1:nmax - if i > nmin - FieldEscapes[i] = (xn > yn ? xf : yf)[i] - else - FieldEscapes[i] = xf[i] ∪ yf[i] + AliasEscapes = xf + elseif isa(xf, FieldEscapes) + if isa(yf, FieldEscapes) + xn, yn = length(xf), length(yf) + nmax, nmin = max(xn, yn), min(xn, yn) + AliasEscapes = Vector{FieldEscape}(undef, nmax) + for i in 1:nmax + if i > nmin + AliasEscapes[i] = (xn > yn ? xf : yf)[i] + else + AliasEscapes[i] = xf[i] ∪ yf[i] + end end + else + AliasEscapes = true # handle conflicting case conservatively + end + else + xf = xf::ArrayEscapes + if isa(yf, ArrayEscapes) + AliasEscapes = xf ∪ yf + else + AliasEscapes = true # handle conflicting case conservatively end end # try to avoid new allocations as minor optimizations @@ -297,7 +358,7 @@ x::EscapeLattice ⊔ y::EscapeLattice = begin x.ReturnEscape | y.ReturnEscape, x.ThrownEscape | y.ThrownEscape, EscapeSites, - FieldEscapes, + AliasEscapes, ) end @@ -312,7 +373,7 @@ x::EscapeLattice ⊓ y::EscapeLattice = begin x.ReturnEscape & y.ReturnEscape, x.ThrownEscape & y.ThrownEscape, x.EscapeSites ∩ y.EscapeSites, - x.FieldEscapes, # FIXME + x.AliasEscapes, # FIXME ) end @@ -362,7 +423,7 @@ Tries to convert analyzable IR element `x::Union{Argument,SSAValue}` to its unique identifier number `xidx` that is valid in the analysis context of `estate`. Returns `nothing` if `x` isn't maintained by `estate` and thus unanalyzable (e.g. `x::GlobalRef`). -`irval` can be used as an inverse function of `iridx`, i.e. +`irval` is the inverse function of `iridx` (not formally), i.e. `irval(iridx(x::Union{Argument,SSAValue}, state), state) === x`. """ function iridx(@nospecialize(x), estate::EscapeState) @@ -383,7 +444,7 @@ end Converts its unique identifier number `xidx` to the original IR element `x::Union{Argument,SSAValue}` that is analyzable in the context of `estate`. -`iridx` can be used as an inverse function of `irval`, i.e. +`iridx` is the inverse function of `irval` (not formally), i.e. `iridx(irval(xidx, state), state) === xidx`. """ function irval(xidx::Int, estate::EscapeState) @@ -453,25 +514,13 @@ function find_escapes(ir::IRCode, nargs::Int) local anyupdate = false for pc in nstmts:-1:1 - stmt = stmts.inst[pc] - - # we escape statements with the `ThrownEscape` property using the effect-freeness - # information computed by the inliner - is_effect_free = stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0 + stmt = stmts[pc][:inst] # collect escape information if isa(stmt, Expr) head = stmt.head if head === :call - has_changes = escape_call!(astate, pc, stmt.args) - # TODO throwness ≠ "effect-free-ness" - if !is_effect_free - for x in stmt.args - add_escape_change!(astate, x, ThrownEscape(pc)) - end - else - has_changes || continue - end + escape_call!(astate, pc, stmt.args) elseif head === :invoke escape_invoke!(astate, pc, stmt.args) elseif head === :new || head === :splatnew @@ -535,11 +584,8 @@ function find_escapes(ir::IRCode, nargs::Int) escape_val!(astate, pc, stmt) elseif isa(stmt, GlobalRef) # global load add_escape_change!(astate, SSAValue(pc), AllEscape()) - elseif isa(stmt, SSAValue) - # NOTE after SROA, we may see SSA value as statement - info = estate[SSAValue(pc)] - add_escape_change!(astate, stmt, info) - add_alias_change!(astate, stmt, SSAValue(pc)) + elseif isa(stmt, SSAValue) # after SROA, we may see SSA value as statement + escape_ssa!(astate, pc, stmt) else @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me continue @@ -635,14 +681,19 @@ function escape_edges!(astate::AnalysisState, pc::Int, edges::Vector{Any}) end end -function escape_val!(astate::AnalysisState, pc::Int, x) - if isdefined(x, :val) - info = astate.estate[SSAValue(pc)] - add_escape_change!(astate, x.val, info) - add_alias_change!(astate, SSAValue(pc), x.val) - end +escape_val!(astate::AnalysisState, pc::Int, x) = + isdefined(x, :val) && _escape_val!(astate, pc, x.val) + +function _escape_val!(astate::AnalysisState, pc::Int, @nospecialize(val)) + ret = SSAValue(pc) + info = astate.estate[ret] + add_escape_change!(astate, val, info) + add_alias_change!(astate, ret, val) end +escape_ssa!(astate::AnalysisState, pc::Int, ssa::SSAValue) = + _escape_val!(astate, pc, ssa) + # NOTE if we don't maintain the alias set that is separated from the lattice state, we can do # soemthing like below: it essentially incorporates forward escape propagation in our default # backward propagation, and leads to inefficient convergence that requires more iterations @@ -659,9 +710,9 @@ end function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) linfo = first(args)::MethodInstance cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) - args = args[2:end] if cache === nothing - for x in args + for i in 2:length(args) + x = args[i] add_escape_change!(astate, x, AllEscape()) end else @@ -669,10 +720,10 @@ function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) retinfo = astate.estate[SSAValue(pc)] # escape information imposed on the call statement method = linfo.def::Method nargs = Int(method.nargs) - for i in 1:length(args) + for i in 2:length(args) arg = args[i] - if i ≤ nargs - argi = i + if i-1 ≤ nargs + argi = i-1 else # handle isva signature: COMBAK will this be invalid once we take alias information into account ? argi = nargs end @@ -700,7 +751,7 @@ function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc # it might be okay from the SROA point of view, since we can't remove the allocation # as far as it's passed to a callee anyway, but still we may want some field analysis # in order to stack allocate it - TOP_FIELD_SETS) + TOP_ALIAS_ESCAPES) if arginfo.EscapeSites === ARGUMENT_ESCAPE_SITES # if this is simply passed as the call argument, we can discard the `ReturnEscape` # information and just propagate the other escape information @@ -723,51 +774,104 @@ end end function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) - objinfo = astate.estate[SSAValue(pc)] + obj = SSAValue(pc) + objinfo = astate.estate[obj] if objinfo == NotAnalyzed() objinfo = NoEscape() end - FieldEscapes = objinfo.FieldEscapes + AliasEscapes = objinfo.AliasEscapes nargs = length(args) - if isa(FieldEscapes, Bool) - # the fields couldn't be analyzed precisely: directly propagate the escape information - # of this object to all its fields (which is the most conservative option) + if isa(AliasEscapes, Bool) + @label conservative_propagation + # the fields couldn't be analyzed precisely: propagate the entire escape information + # of this object to all its fields as the most conservative propagation for i in 2:nargs add_escape_change!(astate, args[i], objinfo) end - else + elseif isa(AliasEscapes, FieldEscapes) # fields are known: propagate escape information imposed on recorded possibilities - nf = length(FieldEscapes) + nf = length(AliasEscapes) for i in 2:nargs + # fields are known: propagate the escape information of this object ignoring field information + add_escape_change!(astate, args[i], ignore_aliasescapes(objinfo)) + # fields are known: propagate escape information imposed on recorded possibilities i-1 > nf && break # may happen when e.g. ϕ-node merges values with different types - escape_field!(astate, args[i], FieldEscapes[i-1]) + escape_field!(astate, args[i], AliasEscapes[i-1]) end + else + # this object has been used as array, but it is allocated as struct here (i.e. should throw) + # update obj's field information and just handle this case conservatively + @assert isa(AliasEscapes, ArrayEscapes) + objinfo = resolve_conflict!(astate, obj, objinfo) + @goto conservative_propagation + end + if !(astate.ir.stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0) + add_thrown_escapes!(astate, pc, args) end end -function escape_field!(astate::AnalysisState, @nospecialize(v), FieldEscape::EscapeSet) +function escape_field!(astate::AnalysisState, @nospecialize(v), xf::FieldEscape) estate = astate.estate - for xidx in FieldEscape + for xidx in xf x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape add_escape_change!(astate, v, estate[x]) add_alias_change!(astate, v, x) end end +function resolve_conflict!(astate::AnalysisState, @nospecialize(obj), objinfo::EscapeLattice) + objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) + add_escape_change!(astate, obj, objinfo) + return objinfo +end + +function add_thrown_escapes!(astate::AnalysisState, pc::Int, args::Vector{Any}, + first_idx::Int = 1, last_idx::Int = length(args)) + for i in first_idx:last_idx + add_escape_change!(astate, args[i], ThrownEscape(pc)) + end +end + # escape every argument `(args[6:length(args[3])])` and the name `args[1]` # TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) + nargs = length(args) + if nargs < 6 + # invalid foreigncall, just escape everything + return add_thrown_escapes!(astate, pc, args) + end foreigncall_nargs = length((args[3])::SimpleVector) name = args[1] - # if normalize(name) === :jl_gc_add_finalizer_th - # # add `FinalizerEscape` ? - # end - add_escape_change!(astate, name, ThrownEscape(pc)) - for i in 6:5+foreigncall_nargs - add_escape_change!(astate, args[i], ThrownEscape(pc)) + nn = normalize(name) + if isa(nn, Symbol) + bounderror_ninds = is_array_resize(nn) + if bounderror_ninds !== nothing + bounderror, ninds = bounderror_ninds + escape_array_resize!(bounderror, ninds, astate, pc, args) + return + end + if is_array_copy(nn) + escape_array_copy!(astate, pc, args) + return + elseif is_array_isassigned(nn) + escape_array_isassigned!(astate, pc, args) + return + end + # if nn === :jl_gc_add_finalizer_th + # # TODO add `FinalizerEscape` ? + # end + end + # NOTE array allocations might have been proven as nothrow (https://github.com/JuliaLang/julia/pull/43565) + if !(astate.ir.stmts[pc][:flag] & IR_FLAG_EFFECT_FREE ≠ 0) + add_escape_change!(astate, name, ThrownEscape(pc)) + for i in 6:5+foreigncall_nargs + add_escape_change!(astate, args[i], ThrownEscape(pc)) + end end end +normalize(@nospecialize x) = isa(x, QuoteNode) ? x.value : x + # NOTE error cases will be handled in `find_escapes` anyway, so we don't need to take care of them below # TODO implement more builtins, make them more accurate # TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? @@ -777,20 +881,32 @@ function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) f = singleton_type(ft) if isa(f, Core.IntrinsicFunction) - return false # COMBAK we may break soundness here, e.g. `pointerref` + # COMBAK we may break soundness and need to account for some aliasing here, e.g. `pointerref` + # argtypes = Any[argextype(args[i], astate.ir) for i = 2:length(args)] + argtypes = Any[] + for i = 2:length(args) + arg = args[i] + push!(argtypes, isexpr(arg, :call) ? Any : argextype(arg, ir)) + end + intrinsic_nothrow(f, argtypes) || add_thrown_escapes!(astate, pc, args, 2) + return end result = escape_builtin!(f, astate, pc, args) - if result === false - return false # nothing to propagate - elseif result === missing + if result === missing # if this call hasn't been handled by any of pre-defined handlers, # we escape this call conservatively for i in 2:length(args) add_escape_change!(astate, args[i], AllEscape()) end - return true - else - return true + return + elseif result === true + return # ThrownEscape is already checked + end + # we escape statements with the `ThrownEscape` property using the effect-freeness + # computed by `stmt_effect_free` invoked within inlining + # TODO throwness ≠ "effect-free-ness" + if !(astate.ir.stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0) + add_thrown_escapes!(astate, pc, args, 2) end end @@ -799,14 +915,14 @@ escape_builtin!(@nospecialize(f), _...) = return missing # safe builtins escape_builtin!(::typeof(isa), _...) = return false escape_builtin!(::typeof(typeof), _...) = return false -escape_builtin!(::typeof(Core.sizeof), _...) = return false +escape_builtin!(::typeof(sizeof), _...) = return false escape_builtin!(::typeof(===), _...) = return false # not really safe, but `ThrownEscape` will be imposed later escape_builtin!(::typeof(isdefined), _...) = return false escape_builtin!(::typeof(throw), _...) = return false -function escape_builtin!(::typeof(Core.ifelse), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) == 4 || return nothing +function escape_builtin!(::typeof(ifelse), astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) == 4 || return false f, cond, th, el = args ret = SSAValue(pc) info = astate.estate[ret] @@ -825,26 +941,26 @@ function escape_builtin!(::typeof(Core.ifelse), astate::AnalysisState, pc::Int, add_escape_change!(astate, el, info) add_alias_change!(astate, el, ret) end - return nothing + return false end function escape_builtin!(::typeof(typeassert), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) == 3 || return nothing + length(args) == 3 || return false f, obj, typ = args ret = SSAValue(pc) info = astate.estate[ret] add_escape_change!(astate, obj, info) add_alias_change!(astate, ret, obj) - return nothing + return false end function escape_builtin!(::typeof(tuple), astate::AnalysisState, pc::Int, args::Vector{Any}) escape_new!(astate, pc, args) - return nothing + return false end function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) ≥ 3 || return nothing + length(args) ≥ 3 || return false ir, estate = astate.ir, astate.estate obj = args[2] typ = widenconst(argextype(obj, ir)) @@ -854,30 +970,35 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg if isa(obj, SSAValue) || isa(obj, Argument) objinfo = estate[obj] else - return + return false end - FieldEscapes = objinfo.FieldEscapes - if isa(FieldEscapes, Bool) - if !FieldEscapes - # the fields of this object aren't analyzed yet: analyze them now + AliasEscapes = objinfo.AliasEscapes + if isa(AliasEscapes, Bool) + if !AliasEscapes + # the fields of this object haven't been analyzed yet: analyze them now nfields = fieldcount_noerror(typ) if nfields !== nothing - FieldEscapes = EscapeSet[EscapeSet() for _ in 1:nfields] - @goto add_field_escape + AliasEscapes = FieldEscape[FieldEscape() for _ in 1:nfields] + @goto record_field_escape end + # unsuccessful field analysis: update obj's field information + objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) + add_escape_change!(astate, obj, objinfo) end - # the field couldn't be analyzed precisely: directly propagate the escape information - # imposed on the return value of this `getfield` call to the object (which is the most conservative option) - # but also with updated field information + @label conservative_propagation + # the field couldn't be analyzed precisely: propagate the escape information + # imposed on the return value of this `getfield` call to the object itself + # as the most conservative propagation ssainfo = estate[SSAValue(pc)] if ssainfo == NotAnalyzed() ssainfo = NoEscape() end - add_escape_change!(astate, obj, EscapeLattice(ssainfo, TOP_FIELD_SETS)) - else - # fields are known: record the return value of this `getfield` call as a possibility that imposes escape - FieldEscapes = copy(FieldEscapes) - @label add_field_escape + add_escape_change!(astate, obj, ssainfo) + elseif isa(AliasEscapes, FieldEscapes) + # fields are known: record the return value of this `getfield` call as a possibility + # that imposes escape on field(s) being referenced + AliasEscapes = copy(AliasEscapes) + @label record_field_escape if isa(typ, DataType) fld = args[3] fldval = try_compute_field(ir, fld) @@ -887,54 +1008,63 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg end if fidx !== nothing # the field is known precisely: propagate this escape information to the field - push!(FieldEscapes[fidx], iridx(SSAValue(pc), estate)) + push!(AliasEscapes[fidx], iridx(SSAValue(pc), estate)) else # the field isn't known precisely: propagate this escape information to all the fields - for FieldEscape in FieldEscapes + for FieldEscape in AliasEscapes push!(FieldEscape, iridx(SSAValue(pc), estate)) end end - add_escape_change!(astate, obj, EscapeLattice(objinfo, FieldEscapes)) + add_escape_change!(astate, obj, EscapeLattice(objinfo, AliasEscapes)) + else + # this object has been used as array, but it is used as struct here (i.e. should throw) + # update obj's field information and just handle this case conservatively + @assert isa(AliasEscapes, ArrayEscapes) + objinfo = resolve_conflict!(astate, obj, objinfo) + @goto conservative_propagation end - return nothing + return false end function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) ≥ 4 || return nothing + length(args) ≥ 4 || return false ir, estate = astate.ir, astate.estate - obj, fld, val = args[2:4] + obj = args[2] + val = args[4] if isa(obj, SSAValue) || isa(obj, Argument) objinfo = estate[obj] else # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively add_escape_change!(astate, val, AllEscape()) - return + return false end - FieldEscapes = objinfo.FieldEscapes - if isa(FieldEscapes, Bool) - if !FieldEscapes - # the fields of this object aren't analyzed yet: analyze them now + AliasEscapes = objinfo.AliasEscapes + if isa(AliasEscapes, Bool) + if !AliasEscapes + # the fields of this object haven't been analyzed yet: analyze them now typ = widenconst(argextype(obj, ir)) nfields = fieldcount_noerror(typ) if nfields !== nothing - # unsuccessful field analysis: update obj's escape information with new field information - FieldEscapes = EscapeSet[EscapeSet() for _ in 1:nfields] - objinfo = EscapeLattice(objinfo, FieldEscapes) + # successful field analysis: update obj's field information + AliasEscapes = FieldEscape[FieldEscape() for _ in 1:nfields] + objinfo = EscapeLattice(objinfo, AliasEscapes) add_escape_change!(astate, obj, objinfo) @goto add_field_escape end - # unsuccessful field analysis: update obj's escape information with new field information - objinfo = EscapeLattice(objinfo, TOP_FIELD_SETS) + # unsuccessful field analysis: update obj's field information + objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) add_escape_change!(astate, obj, objinfo) end - # the field couldn't be analyzed precisely: directly propagate the escape information - # of this object to the field (which is the most conservative option) + @label conservative_propagation + # the field couldn't be analyzed precisely: propagate the entire escape information + # of this object to the value being assigned as the most conservative propagation add_escape_change!(astate, val, objinfo) - else + elseif isa(AliasEscapes, FieldEscapes) # fields are known: propagate escape information imposed on recorded possibilities typ = widenconst(argextype(obj, ir)) @label add_field_escape if isa(typ, DataType) + fld = args[3] fldval = try_compute_field(ir, fld) fidx = try_compute_fieldidx(typ, fldval) else @@ -942,13 +1072,21 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar end if fidx !== nothing # the field is known precisely: propagate this escape information to the field - escape_field!(astate, val, FieldEscapes[fidx]) + escape_field!(astate, val, AliasEscapes[fidx]) else # the field isn't known precisely: propagate this escape information to all the fields - for FieldEscape in FieldEscapes + for FieldEscape in AliasEscapes escape_field!(astate, val, FieldEscape) end end + # fields are known: propagate the escape information of this object ignoring field information + add_escape_change!(astate, val, ignore_aliasescapes(objinfo)) + else + # this object has been used as array, but it is "used" as struct here (i.e. should throw) + # update obj's field information and just handle this case conservatively + @assert isa(AliasEscapes, ArrayEscapes) + objinfo = resolve_conflict!(astate, obj, objinfo) + @goto conservative_propagation end # also propagate escape information imposed on the return value of this `setfield!` ssainfo = estate[SSAValue(pc)] @@ -956,9 +1094,271 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar ssainfo = NoEscape() end add_escape_change!(astate, val, ssainfo) - return nothing + return false +end + +function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) ≥ 4 || return false + # check potential escapes from this arrayref call + # NOTE here we essentially only need to account for TypeError, assuming that + # UndefRefError or BoundsError don't capture any of the arguments here + argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] + boundcheckt = argtypes[1] + aryt = argtypes[2] + if !array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 3) + add_thrown_escapes!(astate, pc, args, 2) + end + # we don't track precise index information about this array and thus don't know what values + # can be referenced here: directly propagate the escape information imposed on the return + # value of this `arrayref` call to the array itself as the most conservative propagation + # but also with updated index information + # TODO enable index analysis when constant values are available? + estate = astate.estate + ary = args[3] + if isa(ary, SSAValue) || isa(ary, Argument) + aryinfo = estate[ary] + else + return true + end + AliasEscapes = aryinfo.AliasEscapes + ret = SSAValue(pc) + if isa(AliasEscapes, Bool) + if !AliasEscapes + # the elements of this array haven't been analyzed yet: set ArrayEscapes now + AliasEscapes = ArrayEscapes() + @goto record_element_escape + end + @label conservative_propagation + ssainfo = estate[ret] + if ssainfo == NotAnalyzed() + ssainfo = NoEscape() + end + add_escape_change!(astate, ary, ssainfo) + if isa(boundcheckt, Const) + if boundcheckt.val::Bool + add_escape_change!(astate, ary, ThrownEscape(pc)) + end + end + elseif isa(AliasEscapes, ArrayEscapes) + # record the return value of this `arrayref` call as a possibility that imposes escape + AliasEscapes = copy(AliasEscapes) + @label record_element_escape + push!(AliasEscapes, iridx(ret, estate)) + if isa(boundcheckt, Const) # record possible BoundsError at this arrayref + if boundcheckt.val::Bool + push!(AliasEscapes, SSAValue(pc)) + end + end + add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) + else + # this object has been used as struct, but it is used as array here (thus should throw) + # update ary's element information and just handle this case conservatively + @assert isa(AliasEscapes, FieldEscapes) + aryinfo = resolve_conflict!(astate, ary, aryinfo) + @goto conservative_propagation + end + return true +end + +function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) ≥ 5 || return false + # check potential escapes from this arrayset call + # NOTE here we essentially only need to account for TypeError, assuming that + # UndefRefError or BoundsError don't capture any of the arguments here + argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] + boundcheckt = argtypes[1] + aryt = argtypes[2] + valt = argtypes[3] + if !(array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 4) && + arrayset_typecheck(aryt, valt)) + add_thrown_escapes!(astate, pc, args, 2) + end + # we don't track precise index information about this array and won't record what value + # is being assigned here: directly propagate the escape information of this array to + # the value being assigned as the most conservative propagation + # TODO enable index analysis when constant values are available? + estate = astate.estate + ary = args[3] + val = args[4] + if isa(ary, SSAValue) || isa(ary, Argument) + aryinfo = estate[ary] + else + # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively + add_escape_change!(astate, val, AllEscape()) + return true + end + AliasEscapes = aryinfo.AliasEscapes + if isa(AliasEscapes, Bool) + if !AliasEscapes + # the elements of this array haven't been analyzed yet: set ArrayEscapes now + AliasEscapes = ArrayEscapes() + @goto add_element_escape + end + @label conservative_propagation + add_escape_change!(astate, val, aryinfo) + elseif isa(AliasEscapes, ArrayEscapes) + @label add_element_escape + for xidx in AliasEscapes + if isa(xidx, Int) + x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape + add_escape_change!(astate, val, estate[x]) + add_alias_change!(astate, val, x) + else + add_escape_change!(astate, val, ThrownEscape(xidx.id)) + end + end + add_escape_change!(astate, val, ignore_aliasescapes(aryinfo)) + else + # this object has been used as struct, but it is "used" as array here (thus should throw) + # update ary's element information and just handle this case conservatively + @assert isa(AliasEscapes, FieldEscapes) + aryinfo = resolve_conflict!(astate, ary, aryinfo) + @goto conservative_propagation + end + # also propagate escape information imposed on the return value of this `arrayset` + ssainfo = estate[SSAValue(pc)] + if ssainfo == NotAnalyzed() + ssainfo = NoEscape() + end + add_escape_change!(astate, ary, ssainfo) + return true end +function escape_builtin!(::typeof(arraysize), astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) == 3 || return false + ary = args[2] + dim = args[3] + if !arraysize_typecheck(ary, dim, astate.ir) + add_escape_change!(astate, ary, ThrownEscape(pc)) + add_escape_change!(astate, dim, ThrownEscape(pc)) + end + # NOTE we may still see "arraysize: dimension out of range", but it doesn't capture anything + return true +end + +function arraysize_typecheck(@nospecialize(ary), @nospecialize(dim), ir::IRCode) + aryt = argextype(ary, ir) + aryt ⊑ₜ Array || return false + dimt = argextype(dim, ir) + dimt ⊑ₜ Int || return false + return true +end + +if isdefined(Core, :arrayfreeze) +function escape_builtin!(::typeof(Core.arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) + return true # TODO needs to account for `TypeError` etc. +end +end # if isdefined(Core, :arrayfreeze) + +# returns nothing if this isn't array resizing operation, +# otherwise returns true if it can throw BoundsError and false if not +function is_array_resize(name::Symbol) + if name === :jl_array_grow_beg || name === :jl_array_grow_end + return false, 1 + elseif name === :jl_array_del_beg || name === :jl_array_del_end + return true, 1 + elseif name === :jl_array_grow_at || name === :jl_array_del_at + return true, 2 + else + return nothing + end +end + +# NOTE may potentially throw "cannot resize array with shared data" error, +# but just ignore it since it doesn't capture anything +function escape_array_resize!(bounderror::Bool, ninds::Int, + astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) ≥ 6+ninds || return add_thrown_escapes!(astate, pc, args) + ary = args[6] + aryt = argextype(ary, astate.ir) + aryt ⊑ₜ Array || return add_thrown_escapes!(astate, pc, args) + for i in 1:ninds + ind = args[i+6] + indt = argextype(ind, astate.ir) + indt ⊑ₜ Integer || return add_thrown_escapes!(astate, pc, args) + end + if bounderror + if isa(ary, SSAValue) || isa(ary, Argument) + estate = astate.estate + aryinfo = estate[ary] + AliasEscapes = aryinfo.AliasEscapes + if isa(AliasEscapes, Bool) + if !AliasEscapes + # the elements of this array haven't been analyzed yet: set ArrayEscapes now + AliasEscapes = ArrayEscapes() + @goto record_element_escape + end + @label conservative_propagation + # array resizing can potentially throw `BoundsError`, impose it now + add_escape_change!(astate, ary, ThrownEscape(pc)) + elseif isa(AliasEscapes, ArrayEscapes) + AliasEscapes = copy(AliasEscapes) + @label record_element_escape + # array resizing can potentially throw `BoundsError`, record it now + push!(AliasEscapes, SSAValue(pc)) + add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) + else + # this object has been used as struct, but it is used as array here (thus should throw) + # update ary's element information and just handle this case conservatively + @assert isa(AliasEscapes, FieldEscapes) + aryinfo = resolve_conflict!(astate, ary, aryinfo) + @goto conservative_propagation + end + end + end +end + +is_array_copy(name::Symbol) = name === :jl_array_copy + +# FIXME this implementation is very conservative, improve the accuracy and solve broken test cases +function escape_array_copy!(astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) ≥ 6 || return add_thrown_escapes!(astate, pc, args) + ary = args[6] + aryt = argextype(ary, astate.ir) + aryt ⊑ₜ Array || return add_thrown_escapes!(astate, pc, args) + if isa(ary, SSAValue) || isa(ary, Argument) + newary = SSAValue(pc) + aryinfo = astate.estate[ary] + newaryinfo = astate.estate[newary] + add_escape_change!(astate, newary, aryinfo) + add_escape_change!(astate, ary, newaryinfo) + end +end + +is_array_isassigned(name::Symbol) = name === :jl_array_isassigned + +function escape_array_isassigned!(astate::AnalysisState, pc::Int, args::Vector{Any}) + if !array_isassigned_nothrow(args, astate.ir) + add_thrown_escapes!(astate, pc, args) + end +end + +function array_isassigned_nothrow(args::Vector{Any}, src::IRCode) + # if !validate_foreigncall_args(args, + # :jl_array_isassigned, Cint, svec(Any,Csize_t), 0, :ccall) + # return false + # end + length(args) ≥ 7 || return false + arytype = argextype(args[6], src) + arytype ⊑ₜ Array || return false + idxtype = argextype(args[7], src) + idxtype ⊑ₜ Csize_t || return false + return true +end + +# # COMBAK do we want to enable this (and also backport this to Base for array allocations?) +# import Core.Compiler: Cint, svec +# function validate_foreigncall_args(args::Vector{Any}, +# name::Symbol, @nospecialize(rt), argtypes::SimpleVector, nreq::Int, convension::Symbol) +# length(args) ≥ 5 || return false +# normalize(args[1]) === name || return false +# args[2] === rt || return false +# args[3] === argtypes || return false +# args[4] === vararg || return false +# normalize(args[5]) === convension || return false +# return true +# end + # NOTE define fancy package utilities when developing EA as an external package if _TOP_MOD !== Core.Compiler include(@__MODULE__, "utils.jl") diff --git a/base/compiler/EscapeAnalysis/utils.jl b/base/compiler/EscapeAnalysis/utils.jl index eba5f076f914b..88afa7de3b60b 100644 --- a/base/compiler/EscapeAnalysis/utils.jl +++ b/base/compiler/EscapeAnalysis/utils.jl @@ -181,7 +181,7 @@ end # register_init_hook!() do import Core: Argument, SSAValue import .CC: widenconst, singleton_type import .EA: - EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_FIELD_SETS, ⊑, ⊏, __clear_escape_cache! + EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES, ⊑, ⊏, __clear_escape_cache! # in order to run a whole analysis from ground zero (e.g. for benchmarking, etc.) __clear_caches!() = (__clear_code_cache!(); __clear_escape_cache!()) @@ -192,9 +192,9 @@ function get_name_color(x::EscapeLattice, symbol::Bool = false) name, color = (getname(EA.NotAnalyzed), "◌"), :plain elseif EA.has_no_escape(x) name, color = (getname(EA.NoEscape), "✓"), :green - elseif EA.NoEscape() ⊏ EA.ignore_fieldsets(x) ⊑ AllReturnEscape() + elseif EA.NoEscape() ⊏ EA.ignore_aliasescapes(x) ⊑ AllReturnEscape() name, color = (getname(EA.ReturnEscape), "↑"), :cyan - elseif EA.NoEscape() ⊏ EA.ignore_fieldsets(x) ⊑ AllThrownEscape() + elseif EA.NoEscape() ⊏ EA.ignore_aliasescapes(x) ⊑ AllThrownEscape() name, color = (getname(EA.ThrownEscape), "↓"), :yellow elseif EA.has_all_escape(x) name, color = (getname(EA.AllEscape), "X"), :red @@ -202,14 +202,14 @@ function get_name_color(x::EscapeLattice, symbol::Bool = false) name, color = (nothing, "*"), :red end name = symbol ? last(name) : first(name) - if name !== nothing && EA.has_fieldsets(x) + if name !== nothing && EA.has_aliasescapes(x) name = string(name, "′") end return name, color end -AllReturnEscape() = EscapeLattice(true, true, false, TOP_ESCAPE_SITES, BOT_FIELD_SETS) -AllThrownEscape() = EscapeLattice(true, false, true, TOP_ESCAPE_SITES, BOT_FIELD_SETS) +AllReturnEscape() = EscapeLattice(true, true, false, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES) +AllThrownEscape() = EscapeLattice(true, false, true, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES) # pcs = sprint(show, collect(x.EscapeSites); context=:limit=>true) function Base.show(io::IO, x::EscapeLattice) From a34166c9d7da6e01df390211d371845319b6128c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 4 Jan 2022 16:37:23 +0900 Subject: [PATCH 16/41] fixup `memory_opt!` --- base/compiler/ssair/passes.jl | 91 ++++++++--------------------------- 1 file changed, 20 insertions(+), 71 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 0742cafe032c0..422c03cacf859 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1391,90 +1391,39 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end -is_array_allocation(stmt::Expr) = _is_known_fcall(stmt, ( - :jl_alloc_array_1d, - :jl_alloc_array_2d, - :jl_alloc_array_3d, - :jl_new_array)) -function _is_known_fcall(stmt::Expr, funcs) - isexpr(stmt, :foreigncall) || return false - s = stmt.args[1] - isa(s, QuoteNode) && (s = s.value) - isa(s, Symbol) || return false - for func in funcs - s === func && return true - end - return false -end - function memory_opt!(ir::IRCode, estate) estate = estate::EscapeAnalysis.EscapeState - compact = IncrementalCompact(ir, false) - # relevant = IdSet{Int}() # allocations - revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in + revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in maybecopies = Int[] # calls to maybecopy - # function mark_escape(@nospecialize val) - # isa(val, SSAValue) || return - # #println(val.id, " escaped.") - # val.id in relevant && pop!(relevant, val.id) - # end - - # function mark_use(val, idx) - # isa(val, SSAValue) || return - # id = val.id - # id in relevant || return - # (haskey(uses, id)) || (uses[id] = Int[]) - # push!(uses[id], idx) - # end - - for ((_, idx), stmt) in compact - (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue - - if is_known_call(stmt, Core.maybecopy, compact) + for idx in 1:length(ir.stmts) + stmt = ir.stmts[idx][:inst] + isexpr(stmt, :call) || continue + if is_known_call(stmt, Core.maybecopy, ir) push!(maybecopies, idx) - # elseif is_array_allocation(stmt) - # push!(relevant, idx) - elseif is_known_call(stmt, Core.arrayfreeze, compact) + elseif is_known_call(stmt, Core.arrayfreeze, ir) if isa(stmt.args[2], SSAValue) push!(revisit, idx) end end end - ir = finish(compact) - isempty(revisit) && isempty(maybecopies) && return ir - - # domtree = construct_domtree(ir.cfg.blocks) - - for idx in revisit - stmt = ir.stmts[idx][:inst]::Expr - arg = stmt.args[2]::SSAValue - # if our escape analysis has determined that this array doesn't escape, we can potentially eliminate an allocation - has_no_escape(estate[arg]) || continue - - # # We're ok to steal the memory if we don't dominate any uses - # ok = true - # if haskey(uses, id) - # for use in uses[id] - # if ssadominates(ir, domtree, idx, use) - # ok = false - # break - # end - # end - # end - # ok || continue - # println("saved an allocation here :", stmt) - stmt.args[1] = Core.mutating_arrayfreeze + if !isempty(revisit) + # if array doesn't escape, we can just change the tag and avoid allocation + for idx in revisit + stmt = ir.stmts[idx][:inst]::Expr + arg = stmt.args[2]::SSAValue + has_no_escape(estate[arg]) || continue + stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) + end end - # TODO: Use escape analysis info to determine if maybecopy should copy - - # for idx in maybecopies - # stmt = ir.stmts[idx][:inst]::Expr - # #println(stmt.args) - # arr = stmt.args[2] - # id = isa(arr, SSAValue) ? arr.id : arr.n # SSAValue or Core.Argument + # if !isempty(maybecopies) + # for idx in maybecopies + # stmt = ir.stmts[idx][:inst]::Expr + # arr = stmt.args[2] + # id = isa(arr, SSAValue) ? arr.id : arr.n # SSAValue or Core.Argument + # end # end return ir From 1b1babf658dac9cf17a5a98dbc1a255c29d83723 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 4 Jan 2022 20:01:30 +0900 Subject: [PATCH 17/41] update EA and handle conflicting field information correctly --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 605b139ad264a..6c86e29a619fb 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -802,7 +802,7 @@ function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) # this object has been used as array, but it is allocated as struct here (i.e. should throw) # update obj's field information and just handle this case conservatively @assert isa(AliasEscapes, ArrayEscapes) - objinfo = resolve_conflict!(astate, obj, objinfo) + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) @goto conservative_propagation end if !(astate.ir.stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0) @@ -819,7 +819,7 @@ function escape_field!(astate::AnalysisState, @nospecialize(v), xf::FieldEscape) end end -function resolve_conflict!(astate::AnalysisState, @nospecialize(obj), objinfo::EscapeLattice) +function escape_unanalyzable_obj!(astate::AnalysisState, @nospecialize(obj), objinfo::EscapeLattice) objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) add_escape_change!(astate, obj, objinfo) return objinfo @@ -982,8 +982,7 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg @goto record_field_escape end # unsuccessful field analysis: update obj's field information - objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) - add_escape_change!(astate, obj, objinfo) + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) end @label conservative_propagation # the field couldn't be analyzed precisely: propagate the escape information @@ -995,9 +994,21 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg end add_escape_change!(astate, obj, ssainfo) elseif isa(AliasEscapes, FieldEscapes) + nfields = fieldcount_noerror(typ) + if nfields === nothing + # unsuccessful field analysis: update obj's field information + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) + @goto conservative_propagation + else + AliasEscapes = copy(AliasEscapes) + if nfields > length(AliasEscapes) + for _ in 1:(nfields-length(AliasEscapes)) + push!(AliasEscapes, FieldEscape()) + end + end + end # fields are known: record the return value of this `getfield` call as a possibility # that imposes escape on field(s) being referenced - AliasEscapes = copy(AliasEscapes) @label record_field_escape if isa(typ, DataType) fld = args[3] @@ -1020,7 +1031,7 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg # this object has been used as array, but it is used as struct here (i.e. should throw) # update obj's field information and just handle this case conservatively @assert isa(AliasEscapes, ArrayEscapes) - objinfo = resolve_conflict!(astate, obj, objinfo) + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) @goto conservative_propagation end return false @@ -1052,16 +1063,26 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar @goto add_field_escape end # unsuccessful field analysis: update obj's field information - objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) - add_escape_change!(astate, obj, objinfo) + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) end @label conservative_propagation # the field couldn't be analyzed precisely: propagate the entire escape information # of this object to the value being assigned as the most conservative propagation add_escape_change!(astate, val, objinfo) elseif isa(AliasEscapes, FieldEscapes) - # fields are known: propagate escape information imposed on recorded possibilities typ = widenconst(argextype(obj, ir)) + nfields = fieldcount_noerror(typ) + if nfields === nothing + # unsuccessful field analysis: update obj's field information + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) + @goto conservative_propagation + elseif nfields > length(AliasEscapes) + AliasEscapes = copy(AliasEscapes) + for _ in 1:(nfields-length(AliasEscapes)) + push!(AliasEscapes, FieldEscape()) + end + end + # fields are known: propagate escape information imposed on recorded possibilities @label add_field_escape if isa(typ, DataType) fld = args[3] @@ -1085,7 +1106,7 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar # this object has been used as array, but it is "used" as struct here (i.e. should throw) # update obj's field information and just handle this case conservatively @assert isa(AliasEscapes, ArrayEscapes) - objinfo = resolve_conflict!(astate, obj, objinfo) + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) @goto conservative_propagation end # also propagate escape information imposed on the return value of this `setfield!` @@ -1154,7 +1175,7 @@ function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, arg # this object has been used as struct, but it is used as array here (thus should throw) # update ary's element information and just handle this case conservatively @assert isa(AliasEscapes, FieldEscapes) - aryinfo = resolve_conflict!(astate, ary, aryinfo) + aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) @goto conservative_propagation end return true @@ -1212,7 +1233,7 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg # this object has been used as struct, but it is "used" as array here (thus should throw) # update ary's element information and just handle this case conservatively @assert isa(AliasEscapes, FieldEscapes) - aryinfo = resolve_conflict!(astate, ary, aryinfo) + aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) @goto conservative_propagation end # also propagate escape information imposed on the return value of this `arrayset` @@ -1301,7 +1322,7 @@ function escape_array_resize!(bounderror::Bool, ninds::Int, # this object has been used as struct, but it is used as array here (thus should throw) # update ary's element information and just handle this case conservatively @assert isa(AliasEscapes, FieldEscapes) - aryinfo = resolve_conflict!(astate, ary, aryinfo) + aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) @goto conservative_propagation end end From eee43bd4e000d7b9d9e489626d48c1238b41a539 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 4 Jan 2022 20:01:46 +0900 Subject: [PATCH 18/41] minor optimization for `memory_opt!` --- base/compiler/ssair/passes.jl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 422c03cacf859..4a677695b99f7 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1393,22 +1393,27 @@ end function memory_opt!(ir::IRCode, estate) estate = estate::EscapeAnalysis.EscapeState - revisit = Int[] # potential targets for a mutating_arrayfreeze drop-in - maybecopies = Int[] # calls to maybecopy + revisit = nothing # potential targets for a mutating_arrayfreeze drop-in + maybecopies = nothing # calls to maybecopy + # mark statements that possibly can be optimized for idx in 1:length(ir.stmts) stmt = ir.stmts[idx][:inst] isexpr(stmt, :call) || continue if is_known_call(stmt, Core.maybecopy, ir) + maybecopies === nothing && (maybecopies = Int[]) push!(maybecopies, idx) elseif is_known_call(stmt, Core.arrayfreeze, ir) + # array as SSA value might have been initialized within this frame + # (thus potentially doesn't escape to anywhere) if isa(stmt.args[2], SSAValue) + revisit === nothing && (revisit = Int[]) push!(revisit, idx) end end end - if !isempty(revisit) + if revisit !== nothing # if array doesn't escape, we can just change the tag and avoid allocation for idx in revisit stmt = ir.stmts[idx][:inst]::Expr @@ -1418,7 +1423,7 @@ function memory_opt!(ir::IRCode, estate) end end - # if !isempty(maybecopies) + # if maybecopies !== nothing # for idx in maybecopies # stmt = ir.stmts[idx][:inst]::Expr # arr = stmt.args[2] From 9b8233a2c25163d98e8548033402c2e7846e219f Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 4 Jan 2022 23:53:13 +0900 Subject: [PATCH 19/41] define tfuncs for `ImmutableArray` primitives --- base/compiler/tfuncs.jl | 43 +++++++++++++++++++++------------ test/compiler/immutablearray.jl | 26 +++++++++++++++++++- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 19874c93c115f..39d6302fcad07 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1448,6 +1448,32 @@ function arrayset_tfunc(@nospecialize(boundscheck), @nospecialize(a), @nospecial end add_tfunc(arrayset, 4, INT_INF, arrayset_tfunc, 20) +# the ImmutableArray operations might involve copies and so their computation costs can be high, +# nevertheless we assign smaller inlining costs to them here, since the escape analysis +# at this moment isn't able to propagate array escapes interprocedurally +# and it will fail to optimize most cases without inlining + +arrayfreeze_tfunc(@nospecialize a) = immutable_array_tfunc(Array, ImmutableArray, a) +add_tfunc(Core.arrayfreeze, 1, 1, arrayfreeze_tfunc, 20) + +mutating_arrayfreeze_tfunc(@nospecialize a) = immutable_array_tfunc(Array, ImmutableArray, a) +add_tfunc(Core.mutating_arrayfreeze, 1, 1, mutating_arrayfreeze_tfunc, 10) + +arraythaw_tfunc(@nospecialize a) = immutable_array_tfunc(ImmutableArray, Array, a) +add_tfunc(Core.arraythaw, 1, 1, arraythaw_tfunc, 20) + +function immutable_array_tfunc(@nospecialize(at), @nospecialize(rt), @nospecialize(a)) + a = widenconst(a) + hasintersect(a, at) || return Bottom + if a <: at + unw = unwrap_unionall(a) + if isa(unw, DataType) + return rewrap_unionall(rt{unw.parameters[1], unw.parameters[2]}, a) + end + end + return rt +end + function _opaque_closure_tfunc(@nospecialize(arg), @nospecialize(isva), @nospecialize(lb), @nospecialize(ub), @nospecialize(source), env::Vector{Any}, linfo::MethodInstance) @@ -1578,22 +1604,7 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp sv::Union{InferenceState,Nothing}) if f === tuple return tuple_tfunc(argtypes) - elseif f === Core.arrayfreeze || f === Core.arraythaw - if length(argtypes) != 1 - return Bottom - end - a = widenconst(argtypes[1]) - at = (f === Core.arrayfreeze ? Array : ImmutableArray) - rt = (f === Core.arrayfreeze ? ImmutableArray : Array) - if a <: at - unw = unwrap_unionall(a) - if isa(unw, DataType) - return rewrap_unionall(rt{unw.parameters[1], unw.parameters[2]}, a) - end - end - return rt - end - if isa(f, IntrinsicFunction) + elseif isa(f, IntrinsicFunction) if is_pure_intrinsic_infer(f) && _all(@nospecialize(a) -> isa(a, Const), argtypes) argvals = anymap(@nospecialize(a) -> (a::Const).val, argtypes) try diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 0c49dd7710f38..3e29ddba4e488 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,4 +1,29 @@ using Test +import Core: ImmutableArray +import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc +const ImmutableVector{T} = Core.ImmutableArray{T,1} + +@test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} +@test arrayfreeze_tfunc(Vector) === ImmutableVector +@test arrayfreeze_tfunc(Array) === ImmutableArray +@test arrayfreeze_tfunc(Any) === ImmutableArray +@test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} +@test arrayfreeze_tfunc(ImmutableVector) === Union{} +@test arrayfreeze_tfunc(ImmutableArray) === Union{} +@test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} +@test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector +@test mutating_arrayfreeze_tfunc(Array) === ImmutableArray +@test mutating_arrayfreeze_tfunc(Any) === ImmutableArray +@test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} +@test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} +@test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} +@test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} +@test arraythaw_tfunc(ImmutableVector) === Vector +@test arraythaw_tfunc(ImmutableArray) === Array +@test arraythaw_tfunc(Any) === Array +@test arraythaw_tfunc(Vector{Int}) === Union{} +@test arraythaw_tfunc(Vector) === Union{} +@test arraythaw_tfunc(Array) === Union{} function test_allocate1() a = Vector{Float64}(undef, 5) @@ -456,4 +481,3 @@ end # sol # end - From 6fad97dbe25f69bb8f12e2535f38f15ab48a57c5 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Fri, 7 Jan 2022 19:49:14 -0500 Subject: [PATCH 20/41] Correct maybecopy and begin implementing its optimization. Also some test cleanup --- base/boot.jl | 8 +- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 1 + base/compiler/ssair/passes.jl | 16 +- src/builtins.c | 9 +- test/compiler/immutablearray.jl | 222 ++++++++++-------- 5 files changed, 136 insertions(+), 120 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index 2813d2aa125f3..9d48231832822 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -274,10 +274,10 @@ struct BoundsError <: Exception a::Any i::Any BoundsError() = new() - # For now, always copy arrays to avoid escaping them - # Eventually, we want to figure out if the copy is needed to save the performance of copying - # (i.e., if a escapes elsewhere, don't bother to make a copy) - + # maybecopy --- non-semantic copy + # if escape analysis proves that this throw is the only place where an object would escape local scope, + # creates a copy to avoid that escape and enable memory optimization through memory_opt! + # otherwise if there are other escapes, maybecopy does not copy and just passes the object BoundsError(@nospecialize(a)) = (@noinline; a isa Array ? new(Core.maybecopy(a)) : new(a)) BoundsError(@nospecialize(a), i) = (@noinline; diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 6c86e29a619fb..e30caf729b694 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -201,6 +201,7 @@ has_return_escape(x::EscapeLattice) = x.ReturnEscape has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites has_thrown_escape(x::EscapeLattice) = x.ThrownEscape has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites +has_only_throw_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x, pc) && length(x.EscapeSites) == 1 has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x ignore_aliasescapes(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_ESCAPES) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 4a677695b99f7..597087ba1429f 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1423,13 +1423,15 @@ function memory_opt!(ir::IRCode, estate) end end - # if maybecopies !== nothing - # for idx in maybecopies - # stmt = ir.stmts[idx][:inst]::Expr - # arr = stmt.args[2] - # id = isa(arr, SSAValue) ? arr.id : arr.n # SSAValue or Core.Argument - # end - # end + if maybecopies !== nothing + for idx in maybecopies + println("analyzing maybecopy at ", pc) + stmt = ir.stmts[idx][:inst]::Expr + arg = stmt.args[2] + has_no_escape(estate[arg]) || continue # XXX is this correct, or has_only_throw_escape(x, pc) where pc is location of throw that created the maybecopy? + stmt.args[1] = GlobalRef(Base, :copy) + end + end return ir end diff --git a/src/builtins.c b/src/builtins.c index abd42b23daec7..f5e74b6355d54 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1403,15 +1403,12 @@ JL_CALLABLE(jl_f_arrayset) JL_CALLABLE(jl_f_maybecopy) { - // maybecopy --- this builtin is never actually supposed to be executed - // instead, calls to it are analyzed and replaced with either a call to copy - // or directly replaced with the object itself that is the target of the maybecopy - // therefore, we just check that there is one argument and do a no-op + // calls to this builtin are potentially replaced with a call to copy + // if not replaced, the default behavior is to typecheck and return the array it was called on JL_NARGS(maybecopy, 1, 1); JL_TYPECHK(maybecopy, array, args[0]); jl_array_t *a = (jl_array_t*)args[0]; - jl_array_t *na = jl_array_copy(a); - return (jl_value_t*)na; + return (jl_value_t*)a; } // type definition ------------------------------------------------------------ diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 3e29ddba4e488..9daaa9ab128bf 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -3,131 +3,147 @@ import Core: ImmutableArray import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc const ImmutableVector{T} = Core.ImmutableArray{T,1} -@test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} -@test arrayfreeze_tfunc(Vector) === ImmutableVector -@test arrayfreeze_tfunc(Array) === ImmutableArray -@test arrayfreeze_tfunc(Any) === ImmutableArray -@test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} -@test arrayfreeze_tfunc(ImmutableVector) === Union{} -@test arrayfreeze_tfunc(ImmutableArray) === Union{} -@test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} -@test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector -@test mutating_arrayfreeze_tfunc(Array) === ImmutableArray -@test mutating_arrayfreeze_tfunc(Any) === ImmutableArray -@test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} -@test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} -@test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} -@test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} -@test arraythaw_tfunc(ImmutableVector) === Vector -@test arraythaw_tfunc(ImmutableArray) === Array -@test arraythaw_tfunc(Any) === Array -@test arraythaw_tfunc(Vector{Int}) === Union{} -@test arraythaw_tfunc(Vector) === Union{} -@test arraythaw_tfunc(Array) === Union{} - -function test_allocate1() - a = Vector{Float64}(undef, 5) - for i = 1:5 - a[i] = i - end - Core.ImmutableArray(a) +@testset "ImmutableArray tfuncs" begin + @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test arrayfreeze_tfunc(Vector) === ImmutableVector + @test arrayfreeze_tfunc(Array) === ImmutableArray + @test arrayfreeze_tfunc(Any) === ImmutableArray + @test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test arrayfreeze_tfunc(ImmutableVector) === Union{} + @test arrayfreeze_tfunc(ImmutableArray) === Union{} + @test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector + @test mutating_arrayfreeze_tfunc(Array) === ImmutableArray + @test mutating_arrayfreeze_tfunc(Any) === ImmutableArray + @test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} + @test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} + @test arraythaw_tfunc(ImmutableVector) === Vector + @test arraythaw_tfunc(ImmutableArray) === Array + @test arraythaw_tfunc(Any) === Array + @test arraythaw_tfunc(Vector{Int}) === Union{} + @test arraythaw_tfunc(Vector) === Union{} + @test arraythaw_tfunc(Array) === Union{} end -function test_allocate2() - a = [1,2,3,4,5] - Core.ImmutableArray(a) -end +@testset "ImmutableArray allocation optimization" begin + @noinline function op(a::AbstractArray) + return reverse(reverse(a)) + end -function test_allocate3() - a = Matrix{Float64}(undef, 5, 2) - for i = 1:5 - for j = 1:2 - a[i, j] = i + j + function allo1() + a = Vector{Float64}(undef, 5) + for i = 1:5 + a[i] = i end + return Core.ImmutableArray(a) end - Core.ImmutableArray(a) -end -function test_allocate4() - a = Core.ImmutableArray{Float64}(undef, 5) -end + function allo2() + a = [1,2,3,4,5] + return Core.ImmutableArray(a) + end -function test_broadcast1() - a = Core.ImmutableArray([1,2,3]) - typeof(a .+ a) <: Core.ImmutableArray -end + function allo3() + a = Matrix{Float64}(undef, 5, 2) + for i = 1:5 + for j = 1:2 + a[i, j] = i + j + end + end + return Core.ImmutableArray(a) + end -function test_allocate5() # test that throwing boundserror doesn't escape - a = [1,2,3] - try - getindex(a, 4) - catch end - Core.ImmutableArray(a) -end + function allo4() # sanity check + return Core.ImmutableArray{Float64}(undef, 5) + end -# function test_maybecopy1() -# a = Vector{Int64}(undef, 5) -# b = Core.maybecopy(a) # doesn't escape in this function - so a !=== b -# @test !(a === b) -# end + function allo5() # test that throwing boundserror doesn't escape + a = [1,2,3] + try + getindex(a, 4) + catch end + return Core.ImmutableArray(a) + end -# function test_maybecopy2() -# a = Vector{Int64}(undef, 5) -# try -# a[6] -# catch e -# @test !(e.a === a) -# end -# end + function allo6() + a = ones(5) + a = op(a) + return Core.ImmutableArray(a) + end -# function test_maybecopy3() -# @noinline function escaper(arr) -# return arr -# end + function test_allo() + # warmup + allo1(); allo2(); allo3(); + allo4(); allo5(); allo6(); + + # these magic values are what the mutable array version would allocate + @test @allocated(allo1()) == 96 + @test @allocated(allo2()) == 96 + @test @allocated(allo3()) == 144 + @test @allocated(allo4()) == 96 + @test @allocated(allo5()) == 160 + @test @allocated(allo6()) == 288 + end -# a = Vector{Int64}(undef, 5) -# escaper(a) -# b = Core.maybecopy(a) -# @test a === b # this time, it does escape, so we give back the actual object -# end + test_allo() +end -# function test_maybecopy4() -# @noinline function escaper(arr) -# return arr -# end +@testset "maybecopy tests" begin + g = nothing # global -# a = Vector{Int64}(undef, 5) -# escaper(a) -# try -# a[6] -# catch e -# if isa(e, BoundsError) -# @test e.a === a # already escaped so we don't copy -# end -# end -# end + @noinline function escape(arr) + g = arr + return arr + end + function mc1() + a = Vector{Int64}(undef, 5) + b = Core.maybecopy(a) # doesn't escape in this function - so a === b + @test a === b + end + # XXX broken until maybecopy implementation is correct + function mc2() + a = Vector{Int64}(undef, 5) + try + getindex(a, 6) + catch e + if isa(e, BoundsError) + @test_broken !(e.a === a) # only escapes through throw, so this should copy + end + end + end + function mc3() + a = Vector{Int64}(undef, 5) + escape(a) + b = Core.maybecopy(a) + @test a === b # escapes elsewhere, so give back the actual object + end -let - # warmup for @allocated - a,b,c,d,e = test_allocate1(), test_allocate2(), test_allocate3(), test_allocate4(), test_allocate5() + function mc4() + a = Vector{Int64}(undef, 5) + escape(a) + try + getindex(a, 6) + catch e + if isa(e, BoundsError) + @test e.a === a # already escaped so we don't copy + end + end + end - # these magic values are ~ what the mutable array version would allocate - @test @allocated(test_allocate1()) < 100 - @test @allocated(test_allocate2()) < 100 - @test @allocated(test_allocate3()) < 150 - @test @allocated(test_allocate4()) < 100 - @test @allocated(test_allocate5()) < 170 - @test test_broadcast1() == true + function test_maybecopy() + mc1(); mc2(); mc3(); + mc4(); + end - # test_maybecopy1() - # test_maybecopy2() - # test_maybecopy3() - # test_maybecopy4() + test_maybecopy() end +@test typeof(Core.ImmutableArray([1,2,3]) .+ Core.ImmutableArray([4,5,6])) <: Core.ImmutableArray # DiffEq Performance Tests From dd77189dda8b1423ef6ce684c89b81cfcaa3c1e5 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 8 Jan 2022 21:16:57 +0900 Subject: [PATCH 21/41] update to latest EA --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 93 ++++++---------- base/compiler/EscapeAnalysis/utils.jl | 100 +++++++++--------- base/compiler/bootstrap.jl | 2 +- base/compiler/optimize.jl | 2 +- 4 files changed, 84 insertions(+), 113 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index e30caf729b694..44d195ed7123c 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -1,7 +1,7 @@ baremodule EscapeAnalysis export - find_escapes, + analyze_escapes, GLOBAL_ESCAPE_CACHE, has_not_analyzed, has_no_escape, @@ -31,53 +31,8 @@ import ._TOP_MOD: # Base definitions import Core.Compiler: # Core.Compiler specific definitions isbitstype, isexpr, is_meta_expr_head, println, IRCode, IR_FLAG_EFFECT_FREE, widenconst, argextype, singleton_type, fieldcount_noerror, - try_compute_fieldidx, hasintersect, ⊑ as ⊑ₜ, intrinsic_nothrow - -if isdefined(Core.Compiler, :try_compute_field) - import Core.Compiler: try_compute_field -else - function try_compute_field(ir::IRCode, @nospecialize(field)) - # fields are usually literals, handle them manually - if isa(field, QuoteNode) - field = field.value - elseif isa(field, Int) || isa(field, Symbol) - # try to resolve other constants, e.g. global reference - else - field = argextype(field, ir) - if isa(field, Const) - field = field.val - else - return nothing - end - end - return isa(field, Union{Int, Symbol}) ? field : nothing - end -end - -if isdefined(Core.Compiler, :array_builtin_common_typecheck) && - isdefined(Core.Compiler, :arrayset_typecheck) - import Core.Compiler: array_builtin_common_typecheck, arrayset_typecheck -else - function array_builtin_common_typecheck( - @nospecialize(boundcheck), @nospecialize(ary), - argtypes::Vector{Any}, first_idx_idx::Int) - (boundcheck ⊑ₜ Bool && ary ⊑ₜ Array) || return false - for i = first_idx_idx:length(argtypes) - argtypes[i] ⊑ₜ Int || return false - end - return true - end - function arrayset_typecheck(@nospecialize(atype), @nospecialize(elm)) - # Check that we can determine the element type - atype = widenconst(atype) - isa(atype, DataType) || return false - ap1 = atype.parameters[1] - isa(ap1, Type) || return false - # Check that the element type is compatible with the element we're assigning - elm ⊑ₜ ap1 || return false - return true - end -end + try_compute_field, try_compute_fieldidx, hasintersect, ⊑ as ⊑ₜ, intrinsic_nothrow, + array_builtin_common_typecheck, arrayset_typecheck, setfield!_nothrow if _TOP_MOD !== Core.Compiler include(@__MODULE__, "disjoint_set.jl") @@ -125,7 +80,7 @@ There are utility constructors to create common `EscapeLattice`s, e.g., - `NoEscape()`: the bottom element of this lattice, meaning it won't escape to anywhere - `AllEscape()`: the topmost element of this lattice, meaning it will escape to everywhere -`find_escapes` will transition these elements from the bottom to the top, +`analyze_escapes` will transition these elements from the bottom to the top, in the same direction as Julia's native type inference routine. An abstract state will be initialized with the bottom(-like) elements: - the call arguments are initialized as `ArgumentReturnEscape()`, because they're visible from a caller immediately @@ -201,7 +156,6 @@ has_return_escape(x::EscapeLattice) = x.ReturnEscape has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites has_thrown_escape(x::EscapeLattice) = x.ThrownEscape has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites -has_only_throw_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x, pc) && length(x.EscapeSites) == 1 has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x ignore_aliasescapes(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_ESCAPES) @@ -496,12 +450,12 @@ struct AnalysisState end """ - find_escapes(ir::IRCode, nargs::Int) -> EscapeState + analyze_escapes(ir::IRCode, nargs::Int) -> EscapeState Analyzes escape information in `ir`. `nargs` is the number of actual arguments of the analyzed call. """ -function find_escapes(ir::IRCode, nargs::Int) +function analyze_escapes(ir::IRCode, nargs::Int) stmts = ir.stmts nstmts = length(stmts) @@ -873,7 +827,7 @@ end normalize(@nospecialize x) = isa(x, QuoteNode) ? x.value : x -# NOTE error cases will be handled in `find_escapes` anyway, so we don't need to take care of them below +# NOTE error cases will be handled in `analyze_escapes` anyway, so we don't need to take care of them below # TODO implement more builtins, make them more accurate # TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? @@ -1048,7 +1002,7 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar else # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively add_escape_change!(astate, val, AllEscape()) - return false + @goto add_thrown_escapes end AliasEscapes = objinfo.AliasEscapes if isa(AliasEscapes, Bool) @@ -1116,7 +1070,14 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar ssainfo = NoEscape() end add_escape_change!(astate, val, ssainfo) - return false + # compute the throwness of this setfield! call here since builtin_nothrow doesn't account for that + @label add_thrown_escapes + argtypes = Any[] + for i = 2:length(args) + push!(argtypes, argextype(args[i], ir)) + end + setfield!_nothrow(argtypes) || add_thrown_escapes!(astate, pc, args, 2) + return true end function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, args::Vector{Any}) @@ -1266,12 +1227,6 @@ function arraysize_typecheck(@nospecialize(ary), @nospecialize(dim), ir::IRCode) return true end -if isdefined(Core, :arrayfreeze) -function escape_builtin!(::typeof(Core.arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) - return true # TODO needs to account for `TypeError` etc. -end -end # if isdefined(Core, :arrayfreeze) - # returns nothing if this isn't array resizing operation, # otherwise returns true if it can throw BoundsError and false if not function is_array_resize(name::Symbol) @@ -1381,6 +1336,22 @@ end # return true # end +if isdefined(Core, :arrayfreeze) && isdefined(Core, :arraythaw) && isdefined(Core, :mutating_arrayfreeze) + +escape_builtin!(::typeof(Core.arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = + escape_immutable_array!(Array, astate, pc, args) +escape_builtin!(::typeof(Core.mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = + escape_immutable_array!(Array, astate, pc, args) +escape_builtin!(::typeof(Core.arraythaw), astate::AnalysisState, pc::Int, args::Vector{Any}) = + escape_immutable_array!(Core.ImmutableArray, astate, pc, args) +function escape_immutable_array!(@nospecialize(arytype), astate::AnalysisState, pc::Int, args::Vector{Any}) + length(args) == 2 || return false + argextype(args[2], astate.ir) ⊑ₜ arytype || return false + return true +end + +end # if isdefined(Core, :arrayfreeze) && isdefined(Core, :arraythaw) && isdefined(Core, :mutating_arrayfreeze) + # NOTE define fancy package utilities when developing EA as an external package if _TOP_MOD !== Core.Compiler include(@__MODULE__, "utils.jl") diff --git a/base/compiler/EscapeAnalysis/utils.jl b/base/compiler/EscapeAnalysis/utils.jl index 88afa7de3b60b..373155c9a326c 100644 --- a/base/compiler/EscapeAnalysis/utils.jl +++ b/base/compiler/EscapeAnalysis/utils.jl @@ -10,26 +10,21 @@ let @doc read(README, String) EA end -let __init_hooks__ = [] - global __init__() = foreach(f->f(), __init_hooks__) - global register_init_hook!(@nospecialize(f)) = push!(__init_hooks__, f) -end - # entries # ------- using InteractiveUtils -macro analyze_escapes(ex0...) - return InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :analyze_escapes, ex0) +macro code_escapes(ex0...) + return InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :code_escapes, ex0) end -function analyze_escapes(@nospecialize(f), @nospecialize(types=Tuple{}); - world = get_world_counter(), - interp = Core.Compiler.NativeInterpreter(world)) +function code_escapes(@nospecialize(f), @nospecialize(types=Tuple{}); + world = get_world_counter(), + interp = Core.Compiler.NativeInterpreter(world)) interp = EscapeAnalyzer(interp) results = code_typed(f, types; optimize=true, world, interp) - isone(length(results)) || throw(ArgumentError("`analyze_escapes` only supports single analysis result")) + isone(length(results)) || throw(ArgumentError("`code_escapes` only supports single analysis result")) return EscapeResult(interp.ir, interp.state, interp.linfo) end @@ -54,14 +49,25 @@ import .CC: may_discard_trees, verbose_stmt_info, code_cache, - get_inference_cache + @timeit, + get_inference_cache, + convert_to_ircode, + slot2reg, + compact!, + ssa_inlining_pass!, + sroa_pass!, + adce_pass!, + type_lift_pass!, + JLOptions, + verify_ir, + verify_linetable # usings import Core: - CodeInstance, MethodInstance + CodeInstance, MethodInstance, CodeInfo import .CC: OptimizationState, IRCode import .EA: - find_escapes, GLOBAL_ESCAPE_CACHE, EscapeCache + analyze_escapes, GLOBAL_ESCAPE_CACHE, EscapeCache mutable struct EscapeAnalyzer{State} <: AbstractInterpreter native::NativeInterpreter @@ -140,40 +146,34 @@ function CC.optimize(interp::EscapeAnalyzer, opt::OptimizationState, params::Opt return CC.finish(interp, opt, params, ir, result) end -# HACK enable copy and paste from Core.Compiler -function run_passes_with_ea end -register_init_hook!() do -@eval CC begin - function $(@__MODULE__).run_passes_with_ea(interp::$EscapeAnalyzer, ci::CodeInfo, sv::OptimizationState) - @timeit "convert" ir = convert_to_ircode(ci, sv) - @timeit "slot2reg" ir = slot2reg(ir, ci, sv) - # TODO: Domsorting can produce an updated domtree - no need to recompute here - @timeit "compact 1" ir = compact!(ir) - @timeit "Inlining" ir = ssa_inlining_pass!(ir, ir.linetable, sv.inlining, ci.propagate_inbounds) - # @timeit "verify 2" verify_ir(ir) - @timeit "compact 2" ir = compact!(ir) - nargs = let def = sv.linfo.def - isa(def, Method) ? Int(def.nargs) : 0 - end - @timeit "collect escape information" state = $find_escapes(ir, nargs) - cacheir = copy(ir) - # cache this result - $setindex!($GLOBAL_ESCAPE_CACHE, $EscapeCache(state, cacheir), sv.linfo) - # return back the result - interp.ir = cacheir - interp.state = state - interp.linfo = sv.linfo - @timeit "SROA" ir = sroa_pass!(ir) - @timeit "ADCE" ir = adce_pass!(ir) - @timeit "type lift" ir = type_lift_pass!(ir) - @timeit "compact 3" ir = compact!(ir) - if JLOptions().debug_level == 2 - @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) - end - return ir +function run_passes_with_ea(interp::EscapeAnalyzer, ci::CodeInfo, sv::OptimizationState) + @timeit "convert" ir = convert_to_ircode(ci, sv) + @timeit "slot2reg" ir = slot2reg(ir, ci, sv) + # TODO: Domsorting can produce an updated domtree - no need to recompute here + @timeit "compact 1" ir = compact!(ir) + @timeit "Inlining" ir = ssa_inlining_pass!(ir, ir.linetable, sv.inlining, ci.propagate_inbounds) + # @timeit "verify 2" verify_ir(ir) + @timeit "compact 2" ir = compact!(ir) + nargs = let def = sv.linfo.def + isa(def, Method) ? Int(def.nargs) : 0 + end + @timeit "collect escape information" state = analyze_escapes(ir, nargs) + cacheir = Core.Compiler.copy(ir) + # cache this result + GLOBAL_ESCAPE_CACHE[sv.linfo] = EscapeCache(state, cacheir) + # return back the result + interp.ir = cacheir + interp.state = state + interp.linfo = sv.linfo + @timeit "SROA" ir = sroa_pass!(ir) + @timeit "ADCE" ir = adce_pass!(ir) + @timeit "type lift" ir = type_lift_pass!(ir) + @timeit "compact 3" ir = compact!(ir) + if JLOptions().debug_level == 2 + @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) end + return ir end -end # register_init_hook!() do # printing # -------- @@ -304,8 +304,8 @@ end end # module EAUtils using .EAUtils: - analyze_escapes, - @analyze_escapes + code_escapes, + @code_escapes export - analyze_escapes, - @analyze_escapes + code_escapes, + @code_escapes diff --git a/base/compiler/bootstrap.jl b/base/compiler/bootstrap.jl index 8e4743a28f149..75ec987656509 100644 --- a/base/compiler/bootstrap.jl +++ b/base/compiler/bootstrap.jl @@ -14,7 +14,7 @@ let fs = Any[ # we first create caches for the optimizer, because they contain many loop constructions # and they're better to not run in interpreter even during bootstrapping - find_escapes, run_passes, + analyze_escapes, run_passes, # then we create caches for inference entries typeinf_ext, typeinf, typeinf_edge, ] diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index eeaeb7826df2d..76e91650d3f8f 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -524,7 +524,7 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) nargs = let def = sv.linfo.def isa(def, Method) ? Int(def.nargs) : 0 end - estate = find_escapes(ir, nargs) + estate = analyze_escapes(ir, nargs) setindex!(GLOBAL_ESCAPE_CACHE, estate.escapes[1:estate.nargs], sv.linfo) @timeit "memory opt" ir = memory_opt!(ir, estate) if JLOptions().debug_level == 2 From 39c925bf1650c5d1e2c55f7d393055c7231cae99 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 8 Jan 2022 21:36:02 +0900 Subject: [PATCH 22/41] fixup `memory_opt!` --- base/compiler/ssair/passes.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 9c0ad6c8d5724..911e23d6df135 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1403,12 +1403,16 @@ function memory_opt!(ir::IRCode, estate) stmt = ir.stmts[idx][:inst] isexpr(stmt, :call) || continue if is_known_call(stmt, Core.maybecopy, ir) - maybecopies === nothing && (maybecopies = Int[]) - push!(maybecopies, idx) + val = stmt.args[2] + if isa(val, Argument) || isa(val, SSAValue) + maybecopies === nothing && (maybecopies = Int[]) + push!(maybecopies, idx) + end elseif is_known_call(stmt, Core.arrayfreeze, ir) # array as SSA value might have been initialized within this frame # (thus potentially doesn't escape to anywhere) - if isa(stmt.args[2], SSAValue) + val = stmt.args[2] + if isa(val, SSAValue) revisit === nothing && (revisit = Int[]) push!(revisit, idx) end @@ -1427,9 +1431,8 @@ function memory_opt!(ir::IRCode, estate) if maybecopies !== nothing for idx in maybecopies - println("analyzing maybecopy at ", pc) stmt = ir.stmts[idx][:inst]::Expr - arg = stmt.args[2] + arg = stmt.args[2]::Union{Argument,SSAValue} has_no_escape(estate[arg]) || continue # XXX is this correct, or has_only_throw_escape(x, pc) where pc is location of throw that created the maybecopy? stmt.args[1] = GlobalRef(Base, :copy) end From fbc2c20144b95d4c93964fa41cacb2502075a574 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sun, 9 Jan 2022 17:32:17 +0900 Subject: [PATCH 23/41] update new array tfuncs to accout for `ImmutableArray` --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 17 +++----- base/compiler/tfuncs.jl | 43 ++++++++++--------- test/compiler/inference.jl | 17 ++++++++ 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 44d195ed7123c..013a927fdb511 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -1080,6 +1080,8 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar return true end +const Arrayish = Union{Array,Core.ImmutableArray} + function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 4 || return false # check potential escapes from this arrayref call @@ -1088,7 +1090,7 @@ function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, arg argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] boundcheckt = argtypes[1] aryt = argtypes[2] - if !array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 3) + if !array_builtin_common_typecheck(Arrayish, boundcheckt, aryt, argtypes, 3) add_thrown_escapes!(astate, pc, args, 2) end # we don't track precise index information about this array and thus don't know what values @@ -1152,7 +1154,7 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg boundcheckt = argtypes[1] aryt = argtypes[2] valt = argtypes[3] - if !(array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 4) && + if !(array_builtin_common_typecheck(Array, boundcheckt, aryt, argtypes, 4) && arrayset_typecheck(aryt, valt)) add_thrown_escapes!(astate, pc, args, 2) end @@ -1173,14 +1175,12 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg AliasEscapes = aryinfo.AliasEscapes if isa(AliasEscapes, Bool) if !AliasEscapes - # the elements of this array haven't been analyzed yet: set ArrayEscapes now - AliasEscapes = ArrayEscapes() - @goto add_element_escape + # the elements of this array haven't been analyzed yet: don't need to consider ArrayEscapes for now + @goto add_ary_escape end @label conservative_propagation add_escape_change!(astate, val, aryinfo) elseif isa(AliasEscapes, ArrayEscapes) - @label add_element_escape for xidx in AliasEscapes if isa(xidx, Int) x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape @@ -1190,6 +1190,7 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg add_escape_change!(astate, val, ThrownEscape(xidx.id)) end end + @label add_ary_escape add_escape_change!(astate, val, ignore_aliasescapes(aryinfo)) else # this object has been used as struct, but it is "used" as array here (thus should throw) @@ -1336,8 +1337,6 @@ end # return true # end -if isdefined(Core, :arrayfreeze) && isdefined(Core, :arraythaw) && isdefined(Core, :mutating_arrayfreeze) - escape_builtin!(::typeof(Core.arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = escape_immutable_array!(Array, astate, pc, args) escape_builtin!(::typeof(Core.mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = @@ -1350,8 +1349,6 @@ function escape_immutable_array!(@nospecialize(arytype), astate::AnalysisState, return true end -end # if isdefined(Core, :arrayfreeze) && isdefined(Core, :arraythaw) && isdefined(Core, :mutating_arrayfreeze) - # NOTE define fancy package utilities when developing EA as an external package if _TOP_MOD !== Core.Compiler include(@__MODULE__, "utils.jl") diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 00e9fa12d05ed..e441bf425dd75 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -461,8 +461,10 @@ add_tfunc(Core._typevar, 3, 3, typevar_tfunc, 100) add_tfunc(applicable, 1, INT_INF, (@nospecialize(f), args...)->Bool, 100) add_tfunc(Core.Intrinsics.arraylen, 1, 1, @nospecialize(x)->Int, 4) +const Arrayish = Union{Array,ImmutableArray} + function arraysize_tfunc(@nospecialize(ary), @nospecialize(dim)) - hasintersect(widenconst(ary), Array) || return Bottom + hasintersect(widenconst(ary), Arrayish) || return Bottom hasintersect(widenconst(dim), Int) || return Bottom return Int end @@ -472,7 +474,7 @@ function arraysize_nothrow(argtypes::Vector{Any}) length(argtypes) == 2 || return false ary = argtypes[1] dim = argtypes[2] - ary ⊑ Array || return false + widenconst(ary) <: Arrayish || return false if isa(dim, Const) dimval = dim.val return isa(dimval, Int) && dimval > 0 @@ -1524,27 +1526,27 @@ function tuple_tfunc(argtypes::Vector{Any}) end arrayref_tfunc(@nospecialize(boundscheck), @nospecialize(ary), @nospecialize idxs...) = - _arrayref_tfunc(boundscheck, ary, idxs) -function _arrayref_tfunc(@nospecialize(boundscheck), @nospecialize(ary), - @nospecialize idxs::Tuple) + _arrayref_tfunc(Arrayish, boundscheck, ary, idxs) +function _arrayref_tfunc(@nospecialize(Arytype), + @nospecialize(boundscheck), @nospecialize(ary), @nospecialize idxs::Tuple) isempty(idxs) && return Bottom - array_builtin_common_errorcheck(boundscheck, ary, idxs) || return Bottom - return array_elmtype(ary) + array_builtin_common_errorcheck(Arytype, boundscheck, ary, idxs) || return Bottom + return array_elmtype(Arytype, ary) end add_tfunc(arrayref, 3, INT_INF, arrayref_tfunc, 20) add_tfunc(const_arrayref, 3, INT_INF, arrayref_tfunc, 20) function arrayset_tfunc(@nospecialize(boundscheck), @nospecialize(ary), @nospecialize(item), @nospecialize idxs...) - hasintersect(widenconst(item), _arrayref_tfunc(boundscheck, ary, idxs)) || return Bottom + hasintersect(widenconst(item), _arrayref_tfunc(Array, boundscheck, ary, idxs)) || return Bottom return ary end add_tfunc(arrayset, 4, INT_INF, arrayset_tfunc, 20) -function array_builtin_common_errorcheck(@nospecialize(boundscheck), @nospecialize(ary), - @nospecialize idxs::Tuple) +function array_builtin_common_errorcheck(@nospecialize(Arytype), + @nospecialize(boundscheck), @nospecialize(ary), @nospecialize idxs::Tuple) hasintersect(widenconst(boundscheck), Bool) || return false - hasintersect(widenconst(ary), Array) || return false + hasintersect(widenconst(ary), Arytype) || return false for i = 1:length(idxs) idx = getfield(idxs, i) idx = isvarargtype(idx) ? unwrapva(idx) : widenconst(idx) @@ -1553,9 +1555,9 @@ function array_builtin_common_errorcheck(@nospecialize(boundscheck), @nospeciali return true end -function array_elmtype(@nospecialize ary) +function array_elmtype(@nospecialize(Arytype), @nospecialize(ary)) a = widenconst(ary) - if !has_free_typevars(a) && a <: Array + if !has_free_typevars(a) && a <: Arytype a0 = a if isa(a, UnionAll) a = unwrap_unionall(a0) @@ -1628,11 +1630,12 @@ function array_type_undefable(@nospecialize(arytype)) end end -function array_builtin_common_nothrow(argtypes::Vector{Any}, first_idx_idx::Int) +function array_builtin_common_nothrow(@nospecialize(Arytype), + argtypes::Vector{Any}, first_idx_idx::Int) length(argtypes) >= 4 || return false boundscheck = argtypes[1] arytype = argtypes[2] - array_builtin_common_typecheck(boundscheck, arytype, argtypes, first_idx_idx) || return false + array_builtin_common_typecheck(Arytype, boundscheck, arytype, argtypes, first_idx_idx) || return false # If we could potentially throw undef ref errors, bail out now. arytype = widenconst(arytype) array_type_undefable(arytype) && return false @@ -1647,12 +1650,12 @@ function array_builtin_common_nothrow(argtypes::Vector{Any}, first_idx_idx::Int) return false end -function array_builtin_common_typecheck( +function array_builtin_common_typecheck(@nospecialize(Arytype), @nospecialize(boundscheck), @nospecialize(arytype), argtypes::Vector{Any}, first_idx_idx::Int) - (boundscheck ⊑ Bool && arytype ⊑ Array) || return false + (widenconst(boundscheck) <: Bool && widenconst(arytype) <: Arytype) || return false for i = first_idx_idx:length(argtypes) - argtypes[i] ⊑ Int || return false + widenconst(argtypes[i]) <: Int || return false end return true end @@ -1671,11 +1674,11 @@ end # Query whether the given builtin is guaranteed not to throw given the argtypes function _builtin_nothrow(@nospecialize(f), argtypes::Array{Any,1}, @nospecialize(rt)) if f === arrayset - array_builtin_common_nothrow(argtypes, 4) || return true + array_builtin_common_nothrow(Array, argtypes, 4) || return true # Additionally check element type compatibility return arrayset_typecheck(argtypes[2], argtypes[3]) elseif f === arrayref || f === const_arrayref - return array_builtin_common_nothrow(argtypes, 3) + return array_builtin_common_nothrow(Arrayish, argtypes, 3) elseif f === arraysize return arraysize_nothrow(argtypes) elseif f === Core._expr diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 35d4e87c736fd..1daa0b8c92b38 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -1542,9 +1542,17 @@ import Core.Compiler: Const, arrayref_tfunc, arrayset_tfunc, arraysize_tfunc @test arrayref_tfunc(Const(true), Vector{Int}, Int, Vararg{Int}) === Int @test arrayref_tfunc(Const(true), Vector{Int}, Vararg{Int}) === Int @test arrayref_tfunc(Const(true), Vector{Int}) === Union{} +@test arrayref_tfunc(Const(true), Core.ImmutableArray{Int,1}, Int) === Int +@test arrayref_tfunc(Const(true), Core.ImmutableArray{<:Integer,1}, Int) === Integer +@test arrayref_tfunc(Const(true), Core.ImmutableArray, Int) === Any +@test arrayref_tfunc(Const(true), Core.ImmutableArray{Int,1}, Int, Vararg{Int}) === Int +@test arrayref_tfunc(Const(true), Core.ImmutableArray{Int,1}, Vararg{Int}) === Int +@test arrayref_tfunc(Const(true), Core.ImmutableArray{Int,1}) === Union{} @test arrayref_tfunc(Const(true), String, Int) === Union{} @test arrayref_tfunc(Const(true), Vector{Int}, Float64) === Union{} @test arrayref_tfunc(Int, Vector{Int}, Int) === Union{} +@test arrayref_tfunc(Const(true), Core.ImmutableArray{Int,1}, Float64) === Union{} +@test arrayref_tfunc(Int, Core.ImmutableArray{Int,1}, Int) === Union{} @test arrayset_tfunc(Const(true), Vector{Int}, Int, Int) === Vector{Int} let ua = Vector{<:Integer} @test arrayset_tfunc(Const(true), ua, Int, Int) === ua @@ -1553,13 +1561,22 @@ end @test arrayset_tfunc(Const(true), Any, Int, Int) === Any @test arrayset_tfunc(Const(true), Vector{String}, String, Int, Vararg{Int}) === Vector{String} @test arrayset_tfunc(Const(true), Vector{String}, String, Vararg{Int}) === Vector{String} +@test arrayset_tfunc(Const(true), Core.ImmutableArray{Int,1}, Int, Int) === Union{} +let ua = Core.ImmutableArray{<:Integer,1} + @test arrayset_tfunc(Const(true), ua, Int, Int) === Union{} +end +@test arrayset_tfunc(Const(true), Core.ImmutableArray, Int, Int) === Union{} +@test arrayset_tfunc(Const(true), Core.ImmutableArray{String,1}, String, Int, Vararg{Int}) === Union{} +@test arrayset_tfunc(Const(true), Core.ImmutableArray{String,1}, String, Vararg{Int}) === Union{} @test arrayset_tfunc(Const(true), Vector{String}, String) === Union{} @test arrayset_tfunc(Const(true), String, Char, Int) === Union{} @test arrayset_tfunc(Const(true), Vector{Int}, Int, Float64) === Union{} @test arrayset_tfunc(Int, Vector{Int}, Int, Int) === Union{} @test arrayset_tfunc(Const(true), Vector{Int}, Float64, Int) === Union{} @test arraysize_tfunc(Vector, Int) === Int +@test arraysize_tfunc(Core.ImmutableArray, Int) === Int @test arraysize_tfunc(Vector, Float64) === Union{} +@test arraysize_tfunc(Core.ImmutableArray, Float64) === Union{} @test arraysize_tfunc(String, Int) === Union{} function f23024(::Type{T}, ::Int) where T From 5d937016697f798da696adc28f5730dedec8b60d Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sun, 9 Jan 2022 17:35:34 +0900 Subject: [PATCH 24/41] simplify `memory_opt!` --- base/compiler/ssair/passes.jl | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 911e23d6df135..93f36b2d8436a 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1395,7 +1395,6 @@ end function memory_opt!(ir::IRCode, estate) estate = estate::EscapeAnalysis.EscapeState - revisit = nothing # potential targets for a mutating_arrayfreeze drop-in maybecopies = nothing # calls to maybecopy # mark statements that possibly can be optimized @@ -1411,24 +1410,15 @@ function memory_opt!(ir::IRCode, estate) elseif is_known_call(stmt, Core.arrayfreeze, ir) # array as SSA value might have been initialized within this frame # (thus potentially doesn't escape to anywhere) - val = stmt.args[2] - if isa(val, SSAValue) - revisit === nothing && (revisit = Int[]) - push!(revisit, idx) + ary = stmt.args[2] + if isa(ary, SSAValue) + # if array doesn't escape, we can just change the tag and avoid allocation + has_no_escape(estate[ary]) || continue + stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) end end end - if revisit !== nothing - # if array doesn't escape, we can just change the tag and avoid allocation - for idx in revisit - stmt = ir.stmts[idx][:inst]::Expr - arg = stmt.args[2]::SSAValue - has_no_escape(estate[arg]) || continue - stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) - end - end - if maybecopies !== nothing for idx in maybecopies stmt = ir.stmts[idx][:inst]::Expr From 697b6f1134f4952ef11e3a9d866613a7587b4b66 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Mon, 10 Jan 2022 15:04:42 +0900 Subject: [PATCH 25/41] simplify --- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 14 +++++------ base/compiler/ssair/passes.jl | 23 ++++++------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 013a927fdb511..66c06e39d79a1 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -799,10 +799,10 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) name = args[1] nn = normalize(name) if isa(nn, Symbol) - bounderror_ninds = is_array_resize(nn) - if bounderror_ninds !== nothing - bounderror, ninds = bounderror_ninds - escape_array_resize!(bounderror, ninds, astate, pc, args) + boundserror_ninds = array_resize_info(nn) + if boundserror_ninds !== nothing + boundserror, ninds = boundserror_ninds + escape_array_resize!(boundserror, ninds, astate, pc, args) return end if is_array_copy(nn) @@ -1230,7 +1230,7 @@ end # returns nothing if this isn't array resizing operation, # otherwise returns true if it can throw BoundsError and false if not -function is_array_resize(name::Symbol) +function array_resize_info(name::Symbol) if name === :jl_array_grow_beg || name === :jl_array_grow_end return false, 1 elseif name === :jl_array_del_beg || name === :jl_array_del_end @@ -1244,7 +1244,7 @@ end # NOTE may potentially throw "cannot resize array with shared data" error, # but just ignore it since it doesn't capture anything -function escape_array_resize!(bounderror::Bool, ninds::Int, +function escape_array_resize!(boundserror::Bool, ninds::Int, astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 6+ninds || return add_thrown_escapes!(astate, pc, args) ary = args[6] @@ -1255,7 +1255,7 @@ function escape_array_resize!(bounderror::Bool, ninds::Int, indt = argextype(ind, astate.ir) indt ⊑ₜ Integer || return add_thrown_escapes!(astate, pc, args) end - if bounderror + if boundserror if isa(ary, SSAValue) || isa(ary, Argument) estate = astate.estate aryinfo = estate[ary] diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 93f36b2d8436a..30356902dbd05 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1401,30 +1401,21 @@ function memory_opt!(ir::IRCode, estate) for idx in 1:length(ir.stmts) stmt = ir.stmts[idx][:inst] isexpr(stmt, :call) || continue - if is_known_call(stmt, Core.maybecopy, ir) - val = stmt.args[2] - if isa(val, Argument) || isa(val, SSAValue) - maybecopies === nothing && (maybecopies = Int[]) - push!(maybecopies, idx) - end - elseif is_known_call(stmt, Core.arrayfreeze, ir) + if is_known_call(stmt, Core.arrayfreeze, ir) # array as SSA value might have been initialized within this frame # (thus potentially doesn't escape to anywhere) + length(stmt.args) ≥ 2 || continue ary = stmt.args[2] if isa(ary, SSAValue) # if array doesn't escape, we can just change the tag and avoid allocation has_no_escape(estate[ary]) || continue stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) end - end - end - - if maybecopies !== nothing - for idx in maybecopies - stmt = ir.stmts[idx][:inst]::Expr - arg = stmt.args[2]::Union{Argument,SSAValue} - has_no_escape(estate[arg]) || continue # XXX is this correct, or has_only_throw_escape(x, pc) where pc is location of throw that created the maybecopy? - stmt.args[1] = GlobalRef(Base, :copy) + elseif is_known_call(stmt, Core.maybecopy, ir) + length(stmt.args) ≥ 2 || continue + ary = stmt.args[2] + has_no_escape(estate[ary]) || continue # XXX is this correct, or has_only_throw_escape(x, pc) where pc is location of throw that created the maybecopy? + stmt.args[1] = GlobalRef(Main.Base, :copy) end end From 85076437feccf0c7ae2c768995504cc4c77203f8 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Mon, 10 Jan 2022 23:23:15 +0900 Subject: [PATCH 26/41] improve test implementations --- test/compiler/immutablearray.jl | 301 +++++++++++++++++++++++++++----- 1 file changed, 254 insertions(+), 47 deletions(-) diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 9daaa9ab128bf..9ad47b1f565c4 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -27,67 +27,274 @@ const ImmutableVector{T} = Core.ImmutableArray{T,1} @test arraythaw_tfunc(Array) === Union{} end -@testset "ImmutableArray allocation optimization" begin - @noinline function op(a::AbstractArray) - return reverse(reverse(a)) - end +# mutating_arrayfreeze optimization +# ================================= - function allo1() - a = Vector{Float64}(undef, 5) - for i = 1:5 - a[i] = i - end - return Core.ImmutableArray(a) +import Core.Compiler: argextype, singleton_type +const EMPTY_SPTYPES = Any[] + +code_typed1(args...; kwargs...) = first(only(code_typed(args...; kwargs...)))::Core.CodeInfo +get_code(args...; kwargs...) = code_typed1(args...; kwargs...).code + +# check if `x` is a statement with a given `head` +isnew(@nospecialize x) = Meta.isexpr(x, :new) + +# check if `x` is a dynamic call of a given function +iscall(y) = @nospecialize(x) -> iscall(y, x) +function iscall((src, f)::Tuple{Core.CodeInfo,Base.Callable}, @nospecialize(x)) + return iscall(x) do @nospecialize x + singleton_type(argextype(x, src, EMPTY_SPTYPES)) === f end +end +iscall(pred::Base.Callable, @nospecialize(x)) = Meta.isexpr(x, :call) && pred(x.args[1]) + +# check if `x` is a statically-resolved call of a function whose name is `sym` +isinvoke(y) = @nospecialize(x) -> isinvoke(y, x) +isinvoke(sym::Symbol, @nospecialize(x)) = isinvoke(mi->mi.def.name===sym, x) +isinvoke(pred::Function, @nospecialize(x)) = Meta.isexpr(x, :invoke) && pred(x.args[1]::Core.MethodInstance) + +function is_array_alloc(@nospecialize x) + Meta.isexpr(x, :foreigncall) || return false + args = x.args + name = args[1] + isa(name, QuoteNode) && (name = name.value) + isa(name, Symbol) || return false + return Core.Compiler.alloc_array_ndims(name) !== nothing +end + +# unescaped examples +# ------------------ + +# simplest -- vector +function unescaped1_1(gen) + a = [1,2,3,4,5] + return gen(a) +end +let src = code_typed1(unescaped1, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped1(identity) + allocated = @allocated unescaped1(identity) + unescaped1(ImmutableArray) + @test allocated == @allocated unescaped1(ImmutableArray) +end - function allo2() - a = [1,2,3,4,5] - return Core.ImmutableArray(a) +# handle matrix etc. (actually this example also requires inter-procedural escape handling) +function unescaped1_2(gen) + a = [1 2 3; 4 5 6] + b = [1 2 3 4 5 6] + return gen(a), gen(b) +end +let src = code_typed1(unescaped1_2, (Type{Core.ImmutableArray},)) + # @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 2 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped1_2(identity) + allocated = @allocated unescaped1_2(identity) + unescaped1_2(ImmutableArray) + @test allocated == @allocated unescaped1_2(ImmutableArray) +end + +# multiple returns don't matter +function unescaped2(gen) + a = [1,2,3,4,5] + return gen(a), gen(a) +end +let src = code_typed1(unescaped2, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 2 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped2(identity) + allocated = @allocated unescaped2(identity) + unescaped2(ImmutableArray) + @test allocated == @allocated unescaped2(ImmutableArray) +end + +# arrayset +function unescaped3_1(gen) + a = Vector{Int}(undef, 5) + for i = 1:5 + a[i] = i end + return gen(a) +end +let src = code_typed1(unescaped3_1, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped3_1(identity) + allocated = @allocated unescaped3_1(identity) + unescaped3_1(ImmutableArray) + @test allocated == @allocated unescaped3_1(ImmutableArray) +end - function allo3() - a = Matrix{Float64}(undef, 5, 2) - for i = 1:5 - for j = 1:2 - a[i, j] = i + j - end +function unescaped3_2(gen) + a = Matrix{Float64}(undef, 5, 2) + for i = 1:5 + for j = 1:2 + a[i, j] = i + j end - return Core.ImmutableArray(a) end + return gen(a) +end +let src = code_typed1(unescaped3_2, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped3_2(identity) + allocated = @allocated unescaped3_2(identity) + unescaped3_2(ImmutableArray) + @test allocated == @allocated unescaped3_2(ImmutableArray) +end - function allo4() # sanity check - return Core.ImmutableArray{Float64}(undef, 5) +# array resize +function unescaped4(gen, n) + a = Int[] + for i = 1:n + push!(a, i) end + return gen(a) +end +let src = code_typed1(unescaped4, (Type{Core.ImmutableArray},Int,)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped4(identity, 42) + allocated = @allocated unescaped4(identity, 42) + unescaped4(ImmutableArray, 42) + @test allocated == @allocated unescaped4(ImmutableArray, 42) +end - function allo5() # test that throwing boundserror doesn't escape - a = [1,2,3] - try - getindex(a, 4) - catch end - return Core.ImmutableArray(a) - end +# inter-procedural +@noinline function same′(a) + return reverse(reverse(a)) +end +function unescaped5(gen) + a = ones(5) + a = same′(a) + return gen(a) +end +let src = code_typed1(unescaped5, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(isinvoke(:same′), src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped5(identity) + allocated = @allocated unescaped5(identity) + unescaped5(ImmutableArray) + @test allocated == @allocated unescaped5(ImmutableArray) +end - function allo6() - a = ones(5) - a = op(a) - return Core.ImmutableArray(a) +# ignore ThrownEscape if it never happens when `arrayfreeze` is called +function unescaped6(gen, n) + a = Int[] + for i = 1:n + push!(a, i) end + n > 100 && throw(a) + return gen(a) +end +let src = code_typed1(unescaped6, (Type{Core.ImmutableArray},Int,)) + @test count(is_array_alloc, src.code) == 1 + @test_broken count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped6(identity, 42) + allocated = @allocated unescaped6(identity, 42) + unescaped6(ImmutableArray, 42) + @test_broken allocated == @allocated unescaped6(ImmutableArray, 42) +end - function test_allo() - # warmup - allo1(); allo2(); allo3(); - allo4(); allo5(); allo6(); - - # these magic values are what the mutable array version would allocate - @test @allocated(allo1()) == 96 - @test @allocated(allo2()) == 96 - @test @allocated(allo3()) == 144 - @test @allocated(allo4()) == 96 - @test @allocated(allo5()) == 160 - @test @allocated(allo6()) == 288 - end +# escaped examples +# ---------------- - test_allo() +const Rx = Ref{Any}() # global memory + +function escaped01(gen) + a = [1,2,3,4,5] + return a, gen(a) +end +let src = code_typed1(escaped01, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 + escaped01(identity) + allocated = @allocated escaped01(identity) + escaped01(ImmutableArray) + local a, b + @test allocated < @allocated a, b = escaped01(ImmutableArray) + @test a !== b + @test !(a isa ImmutableArray) +end + +escaped02(a, gen) = gen(a) +let src = code_typed1(escaped02, (Vector{Int}, Type{ImmutableArray},)) + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 + a = [1,2,3] + escaped02(a, ImmutableArray) + b = escaped02(a, ImmutableArray) + @test a !== b + @test !(a isa ImmutableArray) + @test b isa ImmutableArray +end + +function escaped1(gen) + a = [1,2,3,4,5] + global global_array = a + return gen(a) +end +let src = code_typed1(escaped1, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 + escaped1(identity) + allocated = @allocated escaped1(identity) + escaped1(ImmutableArray) + local a + @test allocated < @allocated a = escaped1(ImmutableArray) + @test global_array !== a + @test !(global_array isa ImmutableArray) +end + +function escaped2(gen) + a = [1,2,3,4,5] + Rx[] = a + return gen(a) +end +let src = code_typed1(escaped2, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 + escaped2(identity) + allocated = @allocated escaped2(identity) + escaped2(ImmutableArray) + local a + @test allocated < @allocated a = escaped2(ImmutableArray) + @test Rx[] !== a + @test !(Rx[] isa ImmutableArray) +end + +function escaped3(gen) + a = [1,2,3,4,5] + try + throw(a) + catch err + global global_array = err + end + return gen(a) +end +let src = code_typed1(escaped3, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 + escaped3(identity) + allocated = @allocated escaped3(identity) + escaped3(ImmutableArray) + local a + @test allocated < @allocated a = escaped3(ImmutableArray) + @test global_array !== a + @test !(global_array isa ImmutableArray) end @testset "maybecopy tests" begin From 77eec88ef74a117865fb07a0369015c5a96e9dd5 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Mon, 10 Jan 2022 14:24:34 -0500 Subject: [PATCH 27/41] Fixup unescaped1_1 test --- test/compiler/immutablearray.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 9ad47b1f565c4..75beddea5a297 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -70,14 +70,14 @@ function unescaped1_1(gen) a = [1,2,3,4,5] return gen(a) end -let src = code_typed1(unescaped1, (Type{Core.ImmutableArray},)) +let src = code_typed1(unescaped1_1, (Type{Core.ImmutableArray},)) @test count(is_array_alloc, src.code) == 1 @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped1(identity) - allocated = @allocated unescaped1(identity) - unescaped1(ImmutableArray) - @test allocated == @allocated unescaped1(ImmutableArray) + unescaped1_1(identity) + allocated = @allocated unescaped1_1(identity) + unescaped1_1(ImmutableArray) + @test allocated == @allocated unescaped1_1(ImmutableArray) end # handle matrix etc. (actually this example also requires inter-procedural escape handling) From c958ac9858884b0a9ecee5635c3888873966c2f8 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Mon, 10 Jan 2022 20:29:24 -0500 Subject: [PATCH 28/41] Cleanup memory_opt, add some tests --- base/compiler/ssair/passes.jl | 9 +- test/compiler/immutablearray.jl | 520 ++++++++------------------------ 2 files changed, 132 insertions(+), 397 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 30356902dbd05..33daa1696fa96 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1393,11 +1393,9 @@ function cfg_simplify!(ir::IRCode) return finish(compact) end +# Inspect calls to arrayfreeze to determine if mutating_arrayfreeze can be safely used instead function memory_opt!(ir::IRCode, estate) estate = estate::EscapeAnalysis.EscapeState - maybecopies = nothing # calls to maybecopy - - # mark statements that possibly can be optimized for idx in 1:length(ir.stmts) stmt = ir.stmts[idx][:inst] isexpr(stmt, :call) || continue @@ -1411,11 +1409,6 @@ function memory_opt!(ir::IRCode, estate) has_no_escape(estate[ary]) || continue stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) end - elseif is_known_call(stmt, Core.maybecopy, ir) - length(stmt.args) ≥ 2 || continue - ary = stmt.args[2] - has_no_escape(estate[ary]) || continue # XXX is this correct, or has_only_throw_escape(x, pc) where pc is location of throw that created the maybecopy? - stmt.args[1] = GlobalRef(Main.Base, :copy) end end diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 75beddea5a297..da1eb27a61bc4 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -27,6 +27,24 @@ const ImmutableVector{T} = Core.ImmutableArray{T,1} @test arraythaw_tfunc(Array) === Union{} end +@testset "ImmutableArray builtins" begin + # basic functionality + let + a = [1,2,3] + b = Core.ImmutableArray(a) + @test Core.arrayfreeze(a) === b + @test Core.mutating_arrayfreeze(a) === b + @test Core.arraythaw(b) == a # arraythaw copies so not === + end + # errors + @test_throws ArgumentError Core.arrayfreeze() + @test_throws ArgumentError Core.arrayfreeze([1,2,3], nothing) + @test_throws TypeError Core.arrayfreeze("not an array") + @test_throws ArgumentError Core.mutating_arrayfreeze() + @test_throws ArgumentError Core.mutating_arrayfreeze([1,2,3], nothing) + @test_throws ArgumentError Core.arraythaw() + @test_throws TypeError Core.arraythaw([1,2,3]) +end # mutating_arrayfreeze optimization # ================================= @@ -205,6 +223,82 @@ let src = code_typed1(unescaped6, (Type{Core.ImmutableArray},Int,)) @test_broken allocated == @allocated unescaped6(ImmutableArray, 42) end +# arrayref +function unescaped7_1(gen) + a = [1,2,3] + b = getindex(a, 2) + return gen(a) +end +let src = code_typed1(unescaped7_1, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped7_1(identity) + allocated = @allocated unescaped7_1(identity) + unescaped7_1(ImmutableArray) + @test allocated == @allocated unescaped7_1(ImmutableArray) +end + +@noinline function ipo_getindex′(a, n) + ele = getindex(a, n) + return ele +end +function unescaped7_2(gen) + a = [1,2,3] + b = ipo_getindex′(a, 2) + return gen(a) +end +let src = code_typed1(unescaped7_2, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(isinvoke(:ipo_getindex′), src.code) == 1 + @test_broken count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped7_2(identity) + allocated = @allocated unescaped7_2(identity) + unescaped7_2(ImmutableArray) + @test_broken allocated == @allocated unescaped7_2(ImmutableArray) +end + +# BoundsError (assumes BoundsError doesn't capture arrays) +function unescaped8_1(gen) + a = [1,2,3] + try + getindex(a, 4) + catch + return gen(a) + end +end +let src = code_typed1(unescaped8_1, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped8_1(identity) + allocated = @allocated unescaped7_1(identity) + unescaped8_1(ImmutableArray) + @test allocated == @allocated unescaped7_1(ImmutableArray) +end + +const g = Ref{Any}() + +function unescaped8_2(gen) + a = [1,2,3] + try + getindex(a, 4) + catch e + g[] = e.a # XXX these tests pass, but this optimization is actually incorrect until BoundsError doesn't escape its objects + return gen(a) + end +end +let src = code_typed1(unescaped8_2, (Type{Core.ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 + unescaped8_2(identity) + allocated = @allocated unescaped7_1(identity) + unescaped8_2(ImmutableArray) + @test allocated == @allocated unescaped7_1(ImmutableArray) +end + # escaped examples # ---------------- @@ -297,410 +391,58 @@ let src = code_typed1(escaped3, (Type{Core.ImmutableArray},)) @test !(global_array isa ImmutableArray) end -@testset "maybecopy tests" begin - g = nothing # global - - @noinline function escape(arr) - g = arr - return arr - end - - function mc1() - a = Vector{Int64}(undef, 5) - b = Core.maybecopy(a) # doesn't escape in this function - so a === b - @test a === b - end - - # XXX broken until maybecopy implementation is correct - function mc2() - a = Vector{Int64}(undef, 5) - try - getindex(a, 6) - catch e - if isa(e, BoundsError) - @test_broken !(e.a === a) # only escapes through throw, so this should copy - end - end - end - - function mc3() - a = Vector{Int64}(undef, 5) - escape(a) - b = Core.maybecopy(a) - @test a === b # escapes elsewhere, so give back the actual object - end - - function mc4() - a = Vector{Int64}(undef, 5) - escape(a) - try - getindex(a, 6) - catch e - if isa(e, BoundsError) - @test e.a === a # already escaped so we don't copy - end - end - end - - function test_maybecopy() - mc1(); mc2(); mc3(); - mc4(); - end - - test_maybecopy() -end - -@test typeof(Core.ImmutableArray([1,2,3]) .+ Core.ImmutableArray([4,5,6])) <: Core.ImmutableArray - -# DiffEq Performance Tests - -# using DifferentialEquations -# using StaticArrays - -# function _build_atsit5_caches(::Type{T}) where {T} - -# cs = SVector{6, T}(0.161, 0.327, 0.9, 0.9800255409045097, 1.0, 1.0) - -# as = SVector{21, T}( -# #=a21=# convert(T,0.161), -# #=a31=# convert(T,-0.008480655492356989), -# #=a32=# convert(T,0.335480655492357), -# #=a41=# convert(T,2.8971530571054935), -# #=a42=# convert(T,-6.359448489975075), -# #=a43=# convert(T,4.3622954328695815), -# #=a51=# convert(T,5.325864828439257), -# #=a52=# convert(T,-11.748883564062828), -# #=a53=# convert(T,7.4955393428898365), -# #=a54=# convert(T,-0.09249506636175525), -# #=a61=# convert(T,5.86145544294642), -# #=a62=# convert(T,-12.92096931784711), -# #=a63=# convert(T,8.159367898576159), -# #=a64=# convert(T,-0.071584973281401), -# #=a65=# convert(T,-0.028269050394068383), -# #=a71=# convert(T,0.09646076681806523), -# #=a72=# convert(T,0.01), -# #=a73=# convert(T,0.4798896504144996), -# #=a74=# convert(T,1.379008574103742), -# #=a75=# convert(T,-3.290069515436081), -# #=a76=# convert(T,2.324710524099774) -# ) - -# btildes = SVector{7,T}( -# convert(T,-0.00178001105222577714), -# convert(T,-0.0008164344596567469), -# convert(T,0.007880878010261995), -# convert(T,-0.1447110071732629), -# convert(T,0.5823571654525552), -# convert(T,-0.45808210592918697), -# convert(T,0.015151515151515152) -# ) -# rs = SVector{22, T}( -# #=r11=# convert(T,1.0), -# #=r12=# convert(T,-2.763706197274826), -# #=r13=# convert(T,2.9132554618219126), -# #=r14=# convert(T,-1.0530884977290216), -# #=r22=# convert(T,0.13169999999999998), -# #=r23=# convert(T,-0.2234), -# #=r24=# convert(T,0.1017), -# #=r32=# convert(T,3.9302962368947516), -# #=r33=# convert(T,-5.941033872131505), -# #=r34=# convert(T,2.490627285651253), -# #=r42=# convert(T,-12.411077166933676), -# #=r43=# convert(T,30.33818863028232), -# #=r44=# convert(T,-16.548102889244902), -# #=r52=# convert(T,37.50931341651104), -# #=r53=# convert(T,-88.1789048947664), -# #=r54=# convert(T,47.37952196281928), -# #=r62=# convert(T,-27.896526289197286), -# #=r63=# convert(T,65.09189467479366), -# #=r64=# convert(T,-34.87065786149661), -# #=r72=# convert(T,1.5), -# #=r73=# convert(T,-4), -# #=r74=# convert(T,2.5), -# ) -# return cs, as, btildes, rs -# end +# @testset "maybecopy tests" begin +# g = nothing # global -# function test_imarrays() -# function lorenz(u, p, t) -# a,b,c = u -# x,y,z = p -# dx_dt = x * (b - a) -# dy_dt = a*(y - c) - b -# dz_dt = a*b - z * c -# res = Vector{Float64}(undef, 3) -# res[1], res[2], res[3] = dx_dt, dy_dt, dz_dt -# Core.ImmutableArray(res) +# @noinline function escape(arr) +# g = arr +# return arr # end -# _u0 = Core.ImmutableArray([1.0, 1.0, 1.0]) -# _tspan = (0.0, 100.0) -# _p = (10.0, 28.0, 8.0/3.0) -# prob = ODEProblem(lorenz, _u0, _tspan, _p) - -# u0 = prob.u0 -# tspan = prob.tspan -# f = prob.f -# p = prob.p - -# dt = 0.1f0 -# saveat = nothing -# save_everystep = true -# abstol = 1f-6 -# reltol = 1f-3 - -# t = tspan[1] -# tf = prob.tspan[2] - -# beta1 = 7/50 -# beta2 = 2/25 -# qmax = 10.0 -# qmin = 1/5 -# gamma = 9/10 -# qoldinit = 1e-4 - -# if saveat === nothing -# ts = Vector{eltype(dt)}(undef,1) -# ts[1] = prob.tspan[1] -# us = Vector{typeof(u0)}(undef,0) -# push!(us,recursivecopy(u0)) -# else -# ts = saveat -# cur_t = 1 -# us = MVector{length(ts),typeof(u0)}(undef) -# if prob.tspan[1] == ts[1] -# cur_t += 1 -# us[1] = u0 -# end +# function mc1() +# a = Vector{Int64}(undef, 5) +# b = Core.maybecopy(a) # doesn't escape in this function - so a === b +# @test a === b # end -# u = u0 -# qold = 1e-4 -# k7 = f(u, p, t) - -# cs, as, btildes, rs = _build_atsit5_caches(eltype(u0)) -# c1, c2, c3, c4, c5, c6 = cs -# a21, a31, a32, a41, a42, a43, a51, a52, a53, a54, -# a61, a62, a63, a64, a65, a71, a72, a73, a74, a75, a76 = as -# btilde1, btilde2, btilde3, btilde4, btilde5, btilde6, btilde7 = btildes - -# # FSAL -# while t < tspan[2] -# uprev = u -# k1 = k7 -# EEst = Inf - -# while EEst > 1 -# dt < 1e-14 && error("dt 1 -# dt = dt/min(inv(qmin),q11/gamma) -# else # EEst <= 1 -# @fastmath q = max(inv(qmax),min(inv(qmin),q/gamma)) -# qold = max(EEst,qoldinit) -# dtold = dt -# dt = dt/q #dtnew -# dt = min(abs(dt),abs(tf-t-dtold)) -# told = t - -# if (tf - t - dtold) < 1e-14 -# t = tf -# else -# t += dtold -# end - -# if saveat === nothing && save_everystep -# push!(us,recursivecopy(u)) -# push!(ts,t) -# else saveat !== nothing -# while cur_t <= length(ts) && ts[cur_t] <= t -# savet = ts[cur_t] -# θ = (savet - told)/dtold -# b1θ, b2θ, b3θ, b4θ, b5θ, b6θ, b7θ = bθs(rs, θ) -# us[cur_t] = uprev + dtold*( -# b1θ*k1 + b2θ*k2 + b3θ*k3 + b4θ*k4 + b5θ*k5 + b6θ*k6 + b7θ*k7) -# cur_t += 1 -# end -# end +# # XXX broken until maybecopy implementation is correct +# function mc2() +# a = Vector{Int64}(undef, 5) +# try +# getindex(a, 6) +# catch e +# if isa(e, BoundsError) +# @test_broken !(e.a === a) # only escapes through throw, so this should copy # end # end # end -# if saveat === nothing && !save_everystep -# push!(us,u) -# push!(ts,t) -# end - -# sol = DiffEqBase.build_solution(prob,Tsit5(),ts,us,calculate_error = false) - -# DiffEqBase.has_analytic(prob.f) && DiffEqBase.calculate_solution_errors!(sol;timeseries_errors=true,dense_errors=false) - -# sol -# end - -# function test_marrays() -# function lorenz(u, p, t) -# a,b,c = u -# x,y,z = p -# dx_dt = x * (b - a) -# dy_dt = a*(y - c) - b -# dz_dt = a*b - z * c -# res = Vector{Float64}(undef, 3) -# res[1], res[2], res[3] = dx_dt, dy_dt, dz_dt -# res +# function mc3() +# a = Vector{Int64}(undef, 5) +# escape(a) +# b = Core.maybecopy(a) +# @test a === b # escapes elsewhere, so give back the actual object # end -# _u0 = [1.0, 1.0, 1.0] -# _tspan = (0.0, 100.0) -# _p = (10.0, 28.0, 8.0/3.0) -# prob = ODEProblem(lorenz, _u0, _tspan, _p) - -# u0 = prob.u0 -# tspan = prob.tspan -# f = prob.f -# p = prob.p - -# dt = 0.1f0 -# saveat = nothing -# save_everystep = true -# abstol = 1f-6 -# reltol = 1f-3 - -# t = tspan[1] -# tf = prob.tspan[2] - -# beta1 = 7/50 -# beta2 = 2/25 -# qmax = 10.0 -# qmin = 1/5 -# gamma = 9/10 -# qoldinit = 1e-4 - -# if saveat === nothing -# ts = Vector{eltype(dt)}(undef,1) -# ts[1] = prob.tspan[1] -# us = Vector{typeof(u0)}(undef,0) -# push!(us,recursivecopy(u0)) -# else -# ts = saveat -# cur_t = 1 -# us = MVector{length(ts),typeof(u0)}(undef) -# if prob.tspan[1] == ts[1] -# cur_t += 1 -# us[1] = u0 -# end -# end - -# u = u0 -# qold = 1e-4 -# k7 = f(u, p, t) - -# cs, as, btildes, rs = _build_atsit5_caches(eltype(u0)) -# c1, c2, c3, c4, c5, c6 = cs -# a21, a31, a32, a41, a42, a43, a51, a52, a53, a54, -# a61, a62, a63, a64, a65, a71, a72, a73, a74, a75, a76 = as -# btilde1, btilde2, btilde3, btilde4, btilde5, btilde6, btilde7 = btildes - -# # FSAL -# while t < tspan[2] -# uprev = u -# k1 = k7 -# EEst = Inf - -# while EEst > 1 -# dt < 1e-14 && error("dt 1 -# dt = dt/min(inv(qmin),q11/gamma) -# else # EEst <= 1 -# @fastmath q = max(inv(qmax),min(inv(qmin),q/gamma)) -# qold = max(EEst,qoldinit) -# dtold = dt -# dt = dt/q #dtnew -# dt = min(abs(dt),abs(tf-t-dtold)) -# told = t - -# if (tf - t - dtold) < 1e-14 -# t = tf -# else -# t += dtold -# end - -# if saveat === nothing && save_everystep -# push!(us,recursivecopy(u)) -# push!(ts,t) -# else saveat !== nothing -# while cur_t <= length(ts) && ts[cur_t] <= t -# savet = ts[cur_t] -# θ = (savet - told)/dtold -# b1θ, b2θ, b3θ, b4θ, b5θ, b6θ, b7θ = bθs(rs, θ) -# us[cur_t] = uprev + dtold*( -# b1θ*k1 + b2θ*k2 + b3θ*k3 + b4θ*k4 + b5θ*k5 + b6θ*k6 + b7θ*k7) -# cur_t += 1 -# end -# end +# function mc4() +# a = Vector{Int64}(undef, 5) +# escape(a) +# try +# getindex(a, 6) +# catch e +# if isa(e, BoundsError) +# @test e.a === a # already escaped so we don't copy # end # end # end -# if saveat === nothing && !save_everystep -# push!(us,u) -# push!(ts,t) +# function test_maybecopy() +# mc1(); mc2(); mc3(); +# mc4(); # end -# sol = DiffEqBase.build_solution(prob,Tsit5(),ts,us,calculate_error = false) - -# DiffEqBase.has_analytic(prob.f) && DiffEqBase.calculate_solution_errors!(sol;timeseries_errors=true,dense_errors=false) - -# sol +# test_maybecopy() # end + +# Check that broadcast precedence is working correctly +@test typeof(Core.ImmutableArray([1,2,3]) .+ Core.ImmutableArray([4,5,6])) <: Core.ImmutableArray \ No newline at end of file From c718fb99b0ec1fc7819837390a20a3cdf393bc75 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 11 Jan 2022 14:24:05 +0900 Subject: [PATCH 29/41] avoid unintended name leak, better naming --- test/compiler/immutablearray.jl | 579 +++++++++++++++++--------------- 1 file changed, 304 insertions(+), 275 deletions(-) diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index da1eb27a61bc4..879e2d0ad46bf 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,8 +1,8 @@ using Test -import Core: ImmutableArray +import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc -const ImmutableVector{T} = Core.ImmutableArray{T,1} +const ImmutableVector{T} = ImmutableArray{T,1} @testset "ImmutableArray tfuncs" begin @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} @test arrayfreeze_tfunc(Vector) === ImmutableVector @@ -31,20 +31,28 @@ end # basic functionality let a = [1,2,3] - b = Core.ImmutableArray(a) - @test Core.arrayfreeze(a) === b - @test Core.mutating_arrayfreeze(a) === b - @test Core.arraythaw(b) == a # arraythaw copies so not === + b = ImmutableArray(a) + @test arrayfreeze(a) === b + @test mutating_arrayfreeze(a) === b + @test arraythaw(b) !== a # arraythaw copies so not === end # errors - @test_throws ArgumentError Core.arrayfreeze() - @test_throws ArgumentError Core.arrayfreeze([1,2,3], nothing) - @test_throws TypeError Core.arrayfreeze("not an array") - @test_throws ArgumentError Core.mutating_arrayfreeze() - @test_throws ArgumentError Core.mutating_arrayfreeze([1,2,3], nothing) - @test_throws ArgumentError Core.arraythaw() - @test_throws TypeError Core.arraythaw([1,2,3]) + a = [1,2,3] + b = ImmutableArray(a) + @test_throws ArgumentError arrayfreeze() + @test_throws ArgumentError arrayfreeze([1,2,3], nothing) + @test_throws TypeError arrayfreeze(b) + @test_throws TypeError arrayfreeze("not an array") + @test_throws ArgumentError mutating_arrayfreeze() + @test_throws ArgumentError mutating_arrayfreeze([1,2,3], nothing) + @test_throws TypeError mutating_arrayfreeze(b) + @test_throws TypeError mutating_arrayfreeze("not an array") + @test_throws ArgumentError arraythaw() + @test_throws ArgumentError arraythaw([1,2,3], nothing) + @test_throws TypeError arraythaw(a) + @test_throws TypeError arraythaw("not an array") end + # mutating_arrayfreeze optimization # ================================= @@ -80,315 +88,336 @@ function is_array_alloc(@nospecialize x) return Core.Compiler.alloc_array_ndims(name) !== nothing end -# unescaped examples -# ------------------ +# optimizable examples +# -------------------- -# simplest -- vector -function unescaped1_1(gen) - a = [1,2,3,4,5] - return gen(a) -end -let src = code_typed1(unescaped1_1, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped1_1(identity) - allocated = @allocated unescaped1_1(identity) - unescaped1_1(ImmutableArray) - @test allocated == @allocated unescaped1_1(ImmutableArray) +let # simplest -- vector + function optimizable(gen) + a = [1,2,3,4,5] + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) + end end -# handle matrix etc. (actually this example also requires inter-procedural escape handling) -function unescaped1_2(gen) - a = [1 2 3; 4 5 6] - b = [1 2 3 4 5 6] - return gen(a), gen(b) -end -let src = code_typed1(unescaped1_2, (Type{Core.ImmutableArray},)) - # @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 2 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped1_2(identity) - allocated = @allocated unescaped1_2(identity) - unescaped1_2(ImmutableArray) - @test allocated == @allocated unescaped1_2(ImmutableArray) +let # handle matrix etc. (actually this example also requires inter-procedural escape handling) + function optimizable(gen) + a = [1 2 3; 4 5 6] + b = [1 2 3 4 5 6] + return gen(a), gen(b) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + # @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 2 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) + end end -# multiple returns don't matter -function unescaped2(gen) - a = [1,2,3,4,5] - return gen(a), gen(a) -end -let src = code_typed1(unescaped2, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 2 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped2(identity) - allocated = @allocated unescaped2(identity) - unescaped2(ImmutableArray) - @test allocated == @allocated unescaped2(ImmutableArray) +let # multiple returns don't matter + function optimizable(gen) + a = [1,2,3,4,5] + return gen(a), gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 2 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) + end end -# arrayset -function unescaped3_1(gen) - a = Vector{Int}(undef, 5) - for i = 1:5 - a[i] = i +let # arrayset + function optimizable1(gen) + a = Vector{Int}(undef, 5) + for i = 1:5 + a[i] = i + end + return gen(a) + end + let src = code_typed1(optimizable1, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable1(identity) + allocated = @allocated optimizable1(identity) + optimizable1(ImmutableArray) + @test allocated == @allocated optimizable1(ImmutableArray) end - return gen(a) -end -let src = code_typed1(unescaped3_1, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped3_1(identity) - allocated = @allocated unescaped3_1(identity) - unescaped3_1(ImmutableArray) - @test allocated == @allocated unescaped3_1(ImmutableArray) -end -function unescaped3_2(gen) - a = Matrix{Float64}(undef, 5, 2) - for i = 1:5 - for j = 1:2 - a[i, j] = i + j + function optimizable2(gen) + a = Matrix{Float64}(undef, 5, 2) + for i = 1:5 + for j = 1:2 + a[i, j] = i + j + end end + return gen(a) + end + let src = code_typed1(optimizable2, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable2(identity) + allocated = @allocated optimizable2(identity) + optimizable2(ImmutableArray) + @test allocated == @allocated optimizable2(ImmutableArray) end - return gen(a) -end -let src = code_typed1(unescaped3_2, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped3_2(identity) - allocated = @allocated unescaped3_2(identity) - unescaped3_2(ImmutableArray) - @test allocated == @allocated unescaped3_2(ImmutableArray) end -# array resize -function unescaped4(gen, n) - a = Int[] - for i = 1:n - push!(a, i) +let # arrayref + function optimizable(gen) + a = [1,2,3] + b = getindex(a, 2) + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) end - return gen(a) end -let src = code_typed1(unescaped4, (Type{Core.ImmutableArray},Int,)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped4(identity, 42) - allocated = @allocated unescaped4(identity, 42) - unescaped4(ImmutableArray, 42) - @test allocated == @allocated unescaped4(ImmutableArray, 42) + +let # array resize + function optimizable(gen, n) + a = Int[] + for i = 1:n + push!(a, i) + end + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},Int,)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity, 42) + allocated = @allocated optimizable(identity, 42) + optimizable(ImmutableArray, 42) + @test allocated == @allocated optimizable(ImmutableArray, 42) + end end -# inter-procedural @noinline function same′(a) return reverse(reverse(a)) end -function unescaped5(gen) - a = ones(5) - a = same′(a) - return gen(a) -end -let src = code_typed1(unescaped5, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(isinvoke(:same′), src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped5(identity) - allocated = @allocated unescaped5(identity) - unescaped5(ImmutableArray) - @test allocated == @allocated unescaped5(ImmutableArray) -end - -# ignore ThrownEscape if it never happens when `arrayfreeze` is called -function unescaped6(gen, n) - a = Int[] - for i = 1:n - push!(a, i) +let # inter-procedural + function optimizable(gen) + a = ones(5) + a = same′(a) + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(isinvoke(:same′), src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) end - n > 100 && throw(a) - return gen(a) -end -let src = code_typed1(unescaped6, (Type{Core.ImmutableArray},Int,)) - @test count(is_array_alloc, src.code) == 1 - @test_broken count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test_broken count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped6(identity, 42) - allocated = @allocated unescaped6(identity, 42) - unescaped6(ImmutableArray, 42) - @test_broken allocated == @allocated unescaped6(ImmutableArray, 42) end -# arrayref -function unescaped7_1(gen) - a = [1,2,3] - b = getindex(a, 2) - return gen(a) -end -let src = code_typed1(unescaped7_1, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped7_1(identity) - allocated = @allocated unescaped7_1(identity) - unescaped7_1(ImmutableArray) - @test allocated == @allocated unescaped7_1(ImmutableArray) +let # ignore ThrownEscape if it never happens when `arrayfreeze` is called + function optimizable(gen, n) + a = Int[] + for i = 1:n + push!(a, i) + end + n > 100 && throw(a) + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},Int,)) + @test count(is_array_alloc, src.code) == 1 + @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity, 42) + allocated = @allocated optimizable(identity, 42) + optimizable(ImmutableArray, 42) + @test_broken allocated == @allocated optimizable(ImmutableArray, 42) + end end - @noinline function ipo_getindex′(a, n) ele = getindex(a, n) return ele end -function unescaped7_2(gen) - a = [1,2,3] - b = ipo_getindex′(a, 2) - return gen(a) -end -let src = code_typed1(unescaped7_2, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(isinvoke(:ipo_getindex′), src.code) == 1 - @test_broken count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test_broken count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped7_2(identity) - allocated = @allocated unescaped7_2(identity) - unescaped7_2(ImmutableArray) - @test_broken allocated == @allocated unescaped7_2(ImmutableArray) -end - -# BoundsError (assumes BoundsError doesn't capture arrays) -function unescaped8_1(gen) - a = [1,2,3] - try - getindex(a, 4) - catch +let # ignore ThrownEscape if it never happens when `arrayfreeze` is called (interprocedural) + function optimizable(gen) + a = [1,2,3] + b = ipo_getindex′(a, 2) return gen(a) end -end -let src = code_typed1(unescaped8_1, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped8_1(identity) - allocated = @allocated unescaped7_1(identity) - unescaped8_1(ImmutableArray) - @test allocated == @allocated unescaped7_1(ImmutableArray) + let src = code_typed1(optimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(isinvoke(:ipo_getindex′), src.code) == 1 + @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test_broken allocated == @allocated optimizable(ImmutableArray) + end end const g = Ref{Any}() +let # BoundsError (assumes BoundsError doesn't capture arrays) + function optimizable1(gen) + a = [1,2,3] + try + getindex(a, 4) + catch + return gen(a) + end + end + let src = code_typed1(optimizable1, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable1(identity) + allocated = @allocated optimizable1(identity) + optimizable1(ImmutableArray) + @test allocated == @allocated optimizable1(ImmutableArray) + end -function unescaped8_2(gen) - a = [1,2,3] - try - getindex(a, 4) - catch e - g[] = e.a # XXX these tests pass, but this optimization is actually incorrect until BoundsError doesn't escape its objects - return gen(a) + function optimizable2(gen) + a = [1,2,3] + try + getindex(a, 4) + catch e + g[] = e.a # XXX these tests pass, but this optimization is actually incorrect until BoundsError doesn't escape its objects + return gen(a) + end + end + let src = code_typed1(optimizable2, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable2(identity) + allocated = @allocated optimizable2(identity) + optimizable2(ImmutableArray) + local ia + @test allocated == @allocated ia = optimizable2(ImmutableArray) + @test_broken g[] !== ia end -end -let src = code_typed1(unescaped8_2, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 0 - unescaped8_2(identity) - allocated = @allocated unescaped7_1(identity) - unescaped8_2(ImmutableArray) - @test allocated == @allocated unescaped7_1(ImmutableArray) end -# escaped examples -# ---------------- +# unoptimizable examples +# ---------------------- const Rx = Ref{Any}() # global memory -function escaped01(gen) - a = [1,2,3,4,5] - return a, gen(a) -end -let src = code_typed1(escaped01, (Type{ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 - escaped01(identity) - allocated = @allocated escaped01(identity) - escaped01(ImmutableArray) - local a, b - @test allocated < @allocated a, b = escaped01(ImmutableArray) - @test a !== b - @test !(a isa ImmutableArray) +let # return escape + function unoptimizable(gen) + a = [1,2,3,4,5] + return a, gen(a) + end + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + unoptimizable(identity) + allocated = @allocated unoptimizable(identity) + unoptimizable(ImmutableArray) + local a, b + @test allocated < @allocated a, b = unoptimizable(ImmutableArray) + @test a !== b + @test !(a isa ImmutableArray) + end end -escaped02(a, gen) = gen(a) -let src = code_typed1(escaped02, (Vector{Int}, Type{ImmutableArray},)) - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 - a = [1,2,3] - escaped02(a, ImmutableArray) - b = escaped02(a, ImmutableArray) - @test a !== b - @test !(a isa ImmutableArray) - @test b isa ImmutableArray +let # arg escape + unoptimizable(a, gen) = gen(a) + let src = code_typed1(unoptimizable, (Vector{Int}, Type{ImmutableArray},)) + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + a = [1,2,3] + unoptimizable(a, ImmutableArray) + b = unoptimizable(a, ImmutableArray) + @test a !== b + @test !(a isa ImmutableArray) + @test b isa ImmutableArray + end end -function escaped1(gen) - a = [1,2,3,4,5] - global global_array = a - return gen(a) -end -let src = code_typed1(escaped1, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 - escaped1(identity) - allocated = @allocated escaped1(identity) - escaped1(ImmutableArray) - local a - @test allocated < @allocated a = escaped1(ImmutableArray) - @test global_array !== a - @test !(global_array isa ImmutableArray) +let # global escape + function unoptimizable(gen) + a = [1,2,3,4,5] + global global_array = a + return gen(a) + end + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + unoptimizable(identity) + allocated = @allocated unoptimizable(identity) + unoptimizable(ImmutableArray) + local a + @test allocated < @allocated a = unoptimizable(ImmutableArray) + @test global_array !== a + @test !(global_array isa ImmutableArray) + end end -function escaped2(gen) - a = [1,2,3,4,5] - Rx[] = a - return gen(a) -end -let src = code_typed1(escaped2, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 - escaped2(identity) - allocated = @allocated escaped2(identity) - escaped2(ImmutableArray) - local a - @test allocated < @allocated a = escaped2(ImmutableArray) - @test Rx[] !== a - @test !(Rx[] isa ImmutableArray) +let # global escape + function unoptimizable(gen) + a = [1,2,3,4,5] + Rx[] = a + return gen(a) + end + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + unoptimizable(identity) + allocated = @allocated unoptimizable(identity) + unoptimizable(ImmutableArray) + local a + @test allocated < @allocated a = unoptimizable(ImmutableArray) + @test Rx[] !== a + @test !(Rx[] isa ImmutableArray) + end end -function escaped3(gen) - a = [1,2,3,4,5] - try - throw(a) - catch err - global global_array = err +let # escapes via exception + function unoptimizable(gen) + a = [1,2,3,4,5] + try + throw(a) + catch err + global global_array = err + end + return gen(a) + end + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + unoptimizable(identity) + allocated = @allocated unoptimizable(identity) + unoptimizable(ImmutableArray) + local a + @test allocated < @allocated a = unoptimizable(ImmutableArray) + @test global_array !== a + @test !(global_array isa ImmutableArray) end - return gen(a) -end -let src = code_typed1(escaped3, (Type{Core.ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, Core.mutating_arrayfreeze)), src.code) == 0 - @test count(iscall((src, Core.arrayfreeze)), src.code) == 1 - escaped3(identity) - allocated = @allocated escaped3(identity) - escaped3(ImmutableArray) - local a - @test allocated < @allocated a = escaped3(ImmutableArray) - @test global_array !== a - @test !(global_array isa ImmutableArray) end # @testset "maybecopy tests" begin @@ -431,7 +460,7 @@ end # getindex(a, 6) # catch e # if isa(e, BoundsError) -# @test e.a === a # already escaped so we don't copy +# @test e.a === a # already unoptimizable_ so we don't copy # end # end # end @@ -445,4 +474,4 @@ end # end # Check that broadcast precedence is working correctly -@test typeof(Core.ImmutableArray([1,2,3]) .+ Core.ImmutableArray([4,5,6])) <: Core.ImmutableArray \ No newline at end of file +@test typeof(ImmutableArray([1,2,3]) .+ ImmutableArray([4,5,6])) <: ImmutableArray From c4c7acf0f789f29da825b4981b3a50fc9c8ce532 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Tue, 11 Jan 2022 20:13:30 -0500 Subject: [PATCH 30/41] Add some basic tests and move non-compiler tests --- test/compiler/immutablearray.jl | 52 --------------------- test/immutablearray.jl | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 52 deletions(-) create mode 100644 test/immutablearray.jl diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 879e2d0ad46bf..c990695af7068 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,57 +1,5 @@ using Test import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw -import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc - -const ImmutableVector{T} = ImmutableArray{T,1} -@testset "ImmutableArray tfuncs" begin - @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} - @test arrayfreeze_tfunc(Vector) === ImmutableVector - @test arrayfreeze_tfunc(Array) === ImmutableArray - @test arrayfreeze_tfunc(Any) === ImmutableArray - @test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} - @test arrayfreeze_tfunc(ImmutableVector) === Union{} - @test arrayfreeze_tfunc(ImmutableArray) === Union{} - @test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} - @test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector - @test mutating_arrayfreeze_tfunc(Array) === ImmutableArray - @test mutating_arrayfreeze_tfunc(Any) === ImmutableArray - @test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} - @test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} - @test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} - @test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} - @test arraythaw_tfunc(ImmutableVector) === Vector - @test arraythaw_tfunc(ImmutableArray) === Array - @test arraythaw_tfunc(Any) === Array - @test arraythaw_tfunc(Vector{Int}) === Union{} - @test arraythaw_tfunc(Vector) === Union{} - @test arraythaw_tfunc(Array) === Union{} -end - -@testset "ImmutableArray builtins" begin - # basic functionality - let - a = [1,2,3] - b = ImmutableArray(a) - @test arrayfreeze(a) === b - @test mutating_arrayfreeze(a) === b - @test arraythaw(b) !== a # arraythaw copies so not === - end - # errors - a = [1,2,3] - b = ImmutableArray(a) - @test_throws ArgumentError arrayfreeze() - @test_throws ArgumentError arrayfreeze([1,2,3], nothing) - @test_throws TypeError arrayfreeze(b) - @test_throws TypeError arrayfreeze("not an array") - @test_throws ArgumentError mutating_arrayfreeze() - @test_throws ArgumentError mutating_arrayfreeze([1,2,3], nothing) - @test_throws TypeError mutating_arrayfreeze(b) - @test_throws TypeError mutating_arrayfreeze("not an array") - @test_throws ArgumentError arraythaw() - @test_throws ArgumentError arraythaw([1,2,3], nothing) - @test_throws TypeError arraythaw(a) - @test_throws TypeError arraythaw("not an array") -end # mutating_arrayfreeze optimization # ================================= diff --git a/test/immutablearray.jl b/test/immutablearray.jl new file mode 100644 index 0000000000000..df17cc9fde988 --- /dev/null +++ b/test/immutablearray.jl @@ -0,0 +1,81 @@ +using Test +import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw +import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc + +@testset "basic ImmutableArray functionality" begin + eltypes = (Float16, Float32, Float64, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128) + for t in eltypes + a = rand(t, rand(1:100), rand(1:10)) + b = ImmutableArray(a) + @test a == b + @test a !== b + @test length(a) == length(b) + for i in 1:length(a) + getindex(a, i) == getindex(b, i) + end + @test size(a) == size(b) + if t in (Float16, Float32, Float64) + # @test_broken sum(a) == sum(b) # issue #43772, sometimes works, sometimes doesn't + else + @test sum(a) == sum(b) + end + @test reverse(a) == reverse(b) + @test ndims(a) == ndims(b) + @test axes(a) == axes(b) + @test strides(a) == strides(b) + @test keys(a) == keys(b) + @test_broken IndexStyle(a) == IndexStyle(b) # ImmutableArray is IndexCartesian whereas Array is IndexLinear - worth looking into + @test_broken eachindex(a) == eachindex(b) + end +end + +const ImmutableVector{T} = ImmutableArray{T,1} +@testset "ImmutableArray tfuncs" begin + @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test arrayfreeze_tfunc(Vector) === ImmutableVector + @test arrayfreeze_tfunc(Array) === ImmutableArray + @test arrayfreeze_tfunc(Any) === ImmutableArray + @test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test arrayfreeze_tfunc(ImmutableVector) === Union{} + @test arrayfreeze_tfunc(ImmutableArray) === Union{} + @test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector + @test mutating_arrayfreeze_tfunc(Array) === ImmutableArray + @test mutating_arrayfreeze_tfunc(Any) === ImmutableArray + @test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} + @test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} + @test arraythaw_tfunc(ImmutableVector) === Vector + @test arraythaw_tfunc(ImmutableArray) === Array + @test arraythaw_tfunc(Any) === Array + @test arraythaw_tfunc(Vector{Int}) === Union{} + @test arraythaw_tfunc(Vector) === Union{} + @test arraythaw_tfunc(Array) === Union{} +end + +@testset "ImmutableArray builtins" begin + # basic functionality + let + a = [1,2,3] + b = ImmutableArray(a) + @test arrayfreeze(a) === b + @test mutating_arrayfreeze(a) === b + @test arraythaw(b) !== a # arraythaw copies so not === + end + # errors + a = [1,2,3] + b = ImmutableArray(a) + @test_throws ArgumentError arrayfreeze() + @test_throws ArgumentError arrayfreeze([1,2,3], nothing) + @test_throws TypeError arrayfreeze(b) + @test_throws TypeError arrayfreeze("not an array") + @test_throws ArgumentError mutating_arrayfreeze() + @test_throws ArgumentError mutating_arrayfreeze([1,2,3], nothing) + @test_throws TypeError mutating_arrayfreeze(b) + @test_throws TypeError mutating_arrayfreeze("not an array") + @test_throws ArgumentError arraythaw() + @test_throws ArgumentError arraythaw([1,2,3], nothing) + @test_throws TypeError arraythaw(a) + @test_throws TypeError arraythaw("not an array") +end \ No newline at end of file From 0d7d81a64134a3f8589a23b39c522042d5ac2b53 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 12 Jan 2022 14:44:13 +0900 Subject: [PATCH 31/41] add test cases https://github.com/aviatesk/EscapeAnalysis.jl/pull/72 should address --- base/compiler/ssair/passes.jl | 1 - test/compiler/immutablearray.jl | 63 ++++++++++++++++++++++++++++++++- test/immutablearray.jl | 26 -------------- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 33daa1696fa96..b945c9cda79a1 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1411,6 +1411,5 @@ function memory_opt!(ir::IRCode, estate) end end end - return ir end diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index c990695af7068..bb6a837880df4 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,5 +1,31 @@ using Test import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw +import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc + +const ImmutableVector{T} = ImmutableArray{T,1} +@testset "ImmutableArray tfuncs" begin + @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test arrayfreeze_tfunc(Vector) === ImmutableVector + @test arrayfreeze_tfunc(Array) === ImmutableArray + @test arrayfreeze_tfunc(Any) === ImmutableArray + @test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test arrayfreeze_tfunc(ImmutableVector) === Union{} + @test arrayfreeze_tfunc(ImmutableArray) === Union{} + @test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} + @test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector + @test mutating_arrayfreeze_tfunc(Array) === ImmutableArray + @test mutating_arrayfreeze_tfunc(Any) === ImmutableArray + @test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} + @test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} + @test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} + @test arraythaw_tfunc(ImmutableVector) === Vector + @test arraythaw_tfunc(ImmutableArray) === Array + @test arraythaw_tfunc(Any) === Array + @test arraythaw_tfunc(Vector{Int}) === Union{} + @test arraythaw_tfunc(Vector) === Union{} + @test arraythaw_tfunc(Array) === Union{} +end # mutating_arrayfreeze optimization # ================================= @@ -8,7 +34,6 @@ import Core.Compiler: argextype, singleton_type const EMPTY_SPTYPES = Any[] code_typed1(args...; kwargs...) = first(only(code_typed(args...; kwargs...)))::Core.CodeInfo -get_code(args...; kwargs...) = code_typed1(args...; kwargs...).code # check if `x` is a statement with a given `head` isnew(@nospecialize x) = Meta.isexpr(x, :new) @@ -224,6 +249,42 @@ let # ignore ThrownEscape if it never happens when `arrayfreeze` is called (inte end end +let # nested case + function optimizable(gen, n) + a = [collect(1:m) for m in 1:n] + for i = 1:n + a[i][1] = i + end + return gen(a) + end + let src = code_typed1(optimizable, (Type{ImmutableArray},Int)) + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity, 100) + allocated = @allocated optimizable(identity, 100) + optimizable(ImmutableArray, 100) + @test allocated == @allocated optimizable(ImmutableArray, 100) + end +end + +# demonstrate alias analysis +broadcast_identity(a) = broadcast(identity, a) +function optimizable_aa(gen, n) # can't be a closure somehow + return collect(1:n) |> + Ref |> Ref |> Ref |> + broadcast_identity |> broadcast_identity |> broadcast_identity |> + gen +end +let src = code_typed1(optimizable_aa, (Type{ImmutableArray},Int)) + @test count(is_array_alloc, src.code) == 1 + @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable_aa(identity, 100) + allocated = @allocated optimizable_aa(identity, 100) + optimizable_aa(ImmutableArray, 100) + @test_broken allocated == @allocated optimizable_aa(ImmutableArray, 100) +end + const g = Ref{Any}() let # BoundsError (assumes BoundsError doesn't capture arrays) function optimizable1(gen) diff --git a/test/immutablearray.jl b/test/immutablearray.jl index df17cc9fde988..e0f3c67ec5312 100644 --- a/test/immutablearray.jl +++ b/test/immutablearray.jl @@ -1,6 +1,5 @@ using Test import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw -import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc @testset "basic ImmutableArray functionality" begin eltypes = (Float16, Float32, Float64, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128) @@ -29,31 +28,6 @@ import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_t end end -const ImmutableVector{T} = ImmutableArray{T,1} -@testset "ImmutableArray tfuncs" begin - @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} - @test arrayfreeze_tfunc(Vector) === ImmutableVector - @test arrayfreeze_tfunc(Array) === ImmutableArray - @test arrayfreeze_tfunc(Any) === ImmutableArray - @test arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} - @test arrayfreeze_tfunc(ImmutableVector) === Union{} - @test arrayfreeze_tfunc(ImmutableArray) === Union{} - @test mutating_arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} - @test mutating_arrayfreeze_tfunc(Vector) === ImmutableVector - @test mutating_arrayfreeze_tfunc(Array) === ImmutableArray - @test mutating_arrayfreeze_tfunc(Any) === ImmutableArray - @test mutating_arrayfreeze_tfunc(ImmutableVector{Int}) === Union{} - @test mutating_arrayfreeze_tfunc(ImmutableVector) === Union{} - @test mutating_arrayfreeze_tfunc(ImmutableArray) === Union{} - @test arraythaw_tfunc(ImmutableVector{Int}) === Vector{Int} - @test arraythaw_tfunc(ImmutableVector) === Vector - @test arraythaw_tfunc(ImmutableArray) === Array - @test arraythaw_tfunc(Any) === Array - @test arraythaw_tfunc(Vector{Int}) === Union{} - @test arraythaw_tfunc(Vector) === Union{} - @test arraythaw_tfunc(Array) === Union{} -end - @testset "ImmutableArray builtins" begin # basic functionality let From 150c4901ad596b065edc889b2639f457d3a9da5f Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Fri, 14 Jan 2022 05:32:21 +0900 Subject: [PATCH 32/41] update EA --- .../EscapeAnalysis/{utils.jl => EAUtils.jl} | 172 ++++-- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 555 +++++++++++------- base/compiler/optimize.jl | 2 +- base/compiler/ssair/passes.jl | 6 +- 4 files changed, 470 insertions(+), 265 deletions(-) rename base/compiler/EscapeAnalysis/{utils.jl => EAUtils.jl} (55%) diff --git a/base/compiler/EscapeAnalysis/utils.jl b/base/compiler/EscapeAnalysis/EAUtils.jl similarity index 55% rename from base/compiler/EscapeAnalysis/utils.jl rename to base/compiler/EscapeAnalysis/EAUtils.jl index 373155c9a326c..295fdeac813bd 100644 --- a/base/compiler/EscapeAnalysis/utils.jl +++ b/base/compiler/EscapeAnalysis/EAUtils.jl @@ -1,29 +1,127 @@ +const EA_AS_PKG = Symbol(@__MODULE__) !== :Base # develop EA as an external package + module EAUtils -import ..EscapeAnalysis: EscapeAnalysis +import ..EA_AS_PKG +if EA_AS_PKG + import ..EscapeAnalysis +else + import Core.Compiler.EscapeAnalysis: EscapeAnalysis + Base.getindex(estate::EscapeAnalysis.EscapeState, @nospecialize(x)) = + Core.Compiler.getindex(estate, x) +end const EA = EscapeAnalysis const CC = Core.Compiler -let - README = normpath(dirname(@__DIR__), "README.md") - include_dependency(README) - @doc read(README, String) EA -end - # entries # ------- -using InteractiveUtils +@static if EA_AS_PKG +import InteractiveUtils: gen_call_with_extracted_types_and_kwargs + +@doc """ + @code_escapes [options...] f(args...) +Evaluates the arguments to the function call, determines its types, and then calls +[`code_escapes`](@ref) on the resulting expression. +As with `@code_typed` and its family, any of `code_escapes` keyword arguments can be given +as the optional arguments like `@code_escpase interp=myinterp myfunc(myargs...)`. +""" macro code_escapes(ex0...) - return InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :code_escapes, ex0) + return gen_call_with_extracted_types_and_kwargs(__module__, :code_escapes, ex0) end - -function code_escapes(@nospecialize(f), @nospecialize(types=Tuple{}); +end # @static if EA_AS_PKG + +""" + code_escapes(f, argtypes=Tuple{}; [world], [interp]) -> result::EscapeResult + code_escapes(tt::Type{<:Tuple}; [world], [interp]) -> result::EscapeResult + +Runs the escape analysis on optimized IR of a genefic function call with the given type signature. +Note that the escape analysis runs after inlining, but before any other optimizations. + +```julia +julia> mutable struct SafeRef{T} + x::T + end + +julia> Base.getindex(x::SafeRef) = x.x; + +julia> Base.isassigned(x::SafeRef) = true; + +julia> get′(x) = isassigned(x) ? x[] : throw(x); + +julia> result = code_escapes((String,String,String)) do s1, s2, s3 + r1 = Ref(s1) + r2 = Ref(s2) + r3 = SafeRef(s3) + try + s1 = get′(r1) + ret = sizeof(s1) + catch err + global g = err # will definitely escape `r1` + end + s2 = get′(r2) # still `r2` doesn't escape fully + s3 = get′(r3) # still `r2` doesn't escape fully + return s2, s3 + end +#3(X _2::String, ↑ _3::String, ↑ _4::String) in Main at REPL[7]:2 +2 X 1 ── %1 = %new(Base.RefValue{String}, _2)::Base.RefValue{String} │╻╷╷ Ref +3 *′ │ %2 = %new(Base.RefValue{String}, _3)::Base.RefValue{String} │╻╷╷ Ref +4 ✓′ └─── %3 = %new(SafeRef{String}, _4)::SafeRef{String} │╻╷ SafeRef +5 ◌ 2 ── %4 = \$(Expr(:enter, #8)) │ + ✓′ │ %5 = ϒ (%3)::SafeRef{String} │ + *′ └─── %6 = ϒ (%2)::Base.RefValue{String} │ +6 ◌ 3 ── %7 = Base.isdefined(%1, :x)::Bool │╻╷ get′ + ◌ └─── goto #5 if not %7 ││ + X 4 ── Base.getfield(%1, :x)::String ││╻ getindex + ◌ └─── goto #6 ││ + ◌ 5 ── Main.throw(%1)::Union{} ││ + ◌ └─── unreachable ││ +7 ◌ 6 ── nothing::typeof(Core.sizeof) │╻ sizeof + ◌ │ nothing::Int64 ││ + ◌ └─── \$(Expr(:leave, 1)) │ + ◌ 7 ── goto #10 │ + ✓′ 8 ── %17 = φᶜ (%5)::SafeRef{String} │ + *′ │ %18 = φᶜ (%6)::Base.RefValue{String} │ + ◌ └─── \$(Expr(:leave, 1)) │ + X 9 ── %20 = \$(Expr(:the_exception))::Any │ +9 ◌ │ (Main.g = %20)::Any │ + ◌ └─── \$(Expr(:pop_exception, :(%4)))::Any │ +11 ✓′ 10 ┄ %23 = φ (#7 => %3, #9 => %17)::SafeRef{String} │ + *′ │ %24 = φ (#7 => %2, #9 => %18)::Base.RefValue{String} │ + ◌ │ %25 = Base.isdefined(%24, :x)::Bool ││╻ isassigned + ◌ └─── goto #12 if not %25 ││ + ↑ 11 ─ %27 = Base.getfield(%24, :x)::String │││╻ getproperty + ◌ └─── goto #13 ││ + ◌ 12 ─ Main.throw(%24)::Union{} ││ + ◌ └─── unreachable ││ +12 ↑ 13 ─ %31 = Base.getfield(%23, :x)::String │╻╷╷ get′ +13 ↑ │ %32 = Core.tuple(%27, %31)::Tuple{String, String} │ + ◌ └─── return %32 │ +``` + +The symbols in the side of each call argument and SSA statements represents the following meaning: +- `◌`: this value is not analyzed because escape information of it won't be used anyway (when the object is `isbitstype` for example) +- `✓`: this value never escapes (`has_no_escape(result.state[x])` holds) +- `↑`: this value can escape to the caller via return (`has_return_escape(result.state[x])` holds) +- `X`: this value can escape to somewhere the escape analysis can't reason about like escapes to a global memory (`has_all_escape(result.state[x])` holds) +- `*`: this value's escape state is between the `ReturnEscape` and `AllEscape` in the `EscapeLattice`, e.g. it has unhandled `ThrownEscape` +and additional `′` indicates that field analysis has been done successfully on that value. + +For testing, escape information of each call argument and SSA value can be inspected programmatically as like: +```julia +julia> result.state[Core.Argument(3)] +ReturnEscape + +julia> result.state[Core.SSAValue(3)] +NoEscape′ +``` +""" +function code_escapes(@nospecialize(args...); world = get_world_counter(), interp = Core.Compiler.NativeInterpreter(world)) interp = EscapeAnalyzer(interp) - results = code_typed(f, types; optimize=true, world, interp) + results = code_typed(args...; optimize=true, world, interp) isone(length(results)) || throw(ArgumentError("`code_escapes` only supports single analysis result")) return EscapeResult(interp.ir, interp.state, interp.linfo) end @@ -67,7 +165,7 @@ import Core: import .CC: OptimizationState, IRCode import .EA: - analyze_escapes, GLOBAL_ESCAPE_CACHE, EscapeCache + analyze_escapes, cache_escapes! mutable struct EscapeAnalyzer{State} <: AbstractInterpreter native::NativeInterpreter @@ -157,10 +255,17 @@ function run_passes_with_ea(interp::EscapeAnalyzer, ci::CodeInfo, sv::Optimizati nargs = let def = sv.linfo.def isa(def, Method) ? Int(def.nargs) : 0 end - @timeit "collect escape information" state = analyze_escapes(ir, nargs) + local state + try + @timeit "collect escape information" state = analyze_escapes(ir, nargs) + catch err + @info "error happened within `analyze_escapes`, insepct `Main.ir` and `Main.nargs`" + @eval Main (ir = $ir; nargs = $nargs) + rethrow(err) + end cacheir = Core.Compiler.copy(ir) # cache this result - GLOBAL_ESCAPE_CACHE[sv.linfo] = EscapeCache(state, cacheir) + cache_escapes!(sv.linfo, state, cacheir) # return back the result interp.ir = cacheir interp.state = state @@ -180,37 +285,33 @@ end import Core: Argument, SSAValue import .CC: widenconst, singleton_type -import .EA: - EscapeLattice, EscapeState, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES, ⊑, ⊏, __clear_escape_cache! +import .EA: EscapeLattice, EscapeState, ⊑, ⊏ # in order to run a whole analysis from ground zero (e.g. for benchmarking, etc.) -__clear_caches!() = (__clear_code_cache!(); __clear_escape_cache!()) +__clear_caches!() = (__clear_code_cache!(); EA.__clear_escape_cache!()) function get_name_color(x::EscapeLattice, symbol::Bool = false) getname(x) = string(nameof(x)) - if x == EA.NotAnalyzed() + if x === EA.⊥ name, color = (getname(EA.NotAnalyzed), "◌"), :plain elseif EA.has_no_escape(x) name, color = (getname(EA.NoEscape), "✓"), :green - elseif EA.NoEscape() ⊏ EA.ignore_aliasescapes(x) ⊑ AllReturnEscape() - name, color = (getname(EA.ReturnEscape), "↑"), :cyan - elseif EA.NoEscape() ⊏ EA.ignore_aliasescapes(x) ⊑ AllThrownEscape() - name, color = (getname(EA.ThrownEscape), "↓"), :yellow elseif EA.has_all_escape(x) name, color = (getname(EA.AllEscape), "X"), :red + elseif EA.NoEscape() ⊏ (EA.ignore_thrownescapes ∘ EA.ignore_aliasescapes)(x) ⊑ EA.AllReturnEscape() + name = (getname(EA.ReturnEscape), "↑") + color = EA.has_thrown_escape(x) ? :yellow : :cyan else - name, color = (nothing, "*"), :red + name = (nothing, "*") + color = EA.has_thrown_escape(x) ? :yellow : :bold end name = symbol ? last(name) : first(name) - if name !== nothing && EA.has_aliasescapes(x) + if name !== nothing && !isa(x.AliasEscapes, Bool) name = string(name, "′") end return name, color end -AllReturnEscape() = EscapeLattice(true, true, false, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES) -AllThrownEscape() = EscapeLattice(true, false, true, TOP_ESCAPE_SITES, BOT_ALIAS_ESCAPES) - # pcs = sprint(show, collect(x.EscapeSites); context=:limit=>true) function Base.show(io::IO, x::EscapeLattice) name, color = get_name_color(x) @@ -255,7 +356,7 @@ function print_with_info(io::IO, arg = state[Argument(i)] i == 1 && continue c, color = get_name_color(arg, true) - printstyled(io, '_', i, "::", ir.argtypes[i], ' ', c; color) + printstyled(io, c, ' ', '_', i, "::", ir.argtypes[i]; color) i ≠ state.nargs && print(io, ", ") end print(io, ')') @@ -303,9 +404,10 @@ end end # module EAUtils -using .EAUtils: - code_escapes, - @code_escapes -export - code_escapes, - @code_escapes +if EA_AS_PKG + using .EAUtils: code_escapes, @code_escapes + export code_escapes, @code_escapes +else + using .EAUtils: code_escapes + export code_escapes +end diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 66c06e39d79a1..3b534f18883e9 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -2,12 +2,12 @@ baremodule EscapeAnalysis export analyze_escapes, - GLOBAL_ESCAPE_CACHE, - has_not_analyzed, + cache_escapes!, has_no_escape, has_return_escape, has_thrown_escape, has_all_escape, + is_load_forwardable, is_sroa_eligible, can_elide_finalizer @@ -22,17 +22,17 @@ import ._TOP_MOD: ==, getindex, setindex! import Core: MethodInstance, Const, Argument, SSAValue, PiNode, PhiNode, UpsilonNode, PhiCNode, ReturnNode, GotoNode, GotoIfNot, SimpleVector, sizeof, ifelse, arrayset, arrayref, - arraysize + arraysize, ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw import ._TOP_MOD: # Base definitions @__MODULE__, @eval, @assert, @nospecialize, @inbounds, @inline, @noinline, @label, @goto, !, !==, !=, ≠, +, -, ≤, <, ≥, >, &, |, include, error, missing, copy, - Vector, BitSet, IdDict, IdSet, ∪, ⊆, ∩, :, length, get, first, last, in, isempty, - isassigned, push!, empty!, max, min, Csize_t + Vector, BitSet, IdDict, IdSet, UnitRange, ∪, ⊆, ∩, :, ∈, ∉, in, length, get, first, last, + isempty, isassigned, pop!, push!, empty!, max, min, Csize_t import Core.Compiler: # Core.Compiler specific definitions isbitstype, isexpr, is_meta_expr_head, println, IRCode, IR_FLAG_EFFECT_FREE, widenconst, argextype, singleton_type, fieldcount_noerror, try_compute_field, try_compute_fieldidx, hasintersect, ⊑ as ⊑ₜ, intrinsic_nothrow, - array_builtin_common_typecheck, arrayset_typecheck, setfield!_nothrow + array_builtin_common_typecheck, arrayset_typecheck, setfield!_nothrow, compute_trycatch if _TOP_MOD !== Core.Compiler include(@__MODULE__, "disjoint_set.jl") @@ -45,20 +45,19 @@ const FieldEscape = BitSet const FieldEscapes = Vector{BitSet} # for x in ArrayEscapes: # - x::Int: `irval(x, estate)` imposes escapes on the array elements -# - x::SSAValue: SSA statement (x.id) can potentially escape array elements via BoundsError +# - x::SSAValue: array elements can potentially escape via BoundsError at this SSA statement const ArrayEscapes = IdSet{Union{Int,SSAValue}} """ x::EscapeLattice A lattice for escape information, which holds the following properties: -- `x.Analyzed::Bool`: not formally part of the lattice, indicates `x` has not been analyzed at all -- `x.ReturnEscape::Bool`: indicates `x` may escape to the caller via return, - where `x.ReturnEscape && 0 ∈ x.EscapeSites` has the special meaning that it's visible to - the caller simply because it's passed as call argument -- `x.ThrownEscape::Bool`: indicates `x` may escape to somewhere through an exception -- `x.EscapeSites::BitSet`: records SSA statements where `x` can escape via any of - `ReturnEscape` or `ThrownEscape` +- `x.Analyzed::Bool`: not formally part of the lattice, only indicates `x` has not been analyzed or not +- `x.ReturnEscape::BitSet`: records SSA statements where `x` can escape to the caller via return + where `0 ∈ x.ReturnEscape` has the special meaning that it's visible to the caller + simply because it's passed as call argument +- `x.ThrownEscape::BitSet`: records SSA statements where `x` can be thrown as exception: + this information will be used by `escape_exception!` to propagate potential escapes via exception - `x.AliasEscapes::Union{FieldEscapes,ArrayEscapes,Bool}`: maintains all possible values that may escape objects that can be referenced from `x`: * `x.AliasEscapes === false` indicates the fields/elements of `x` isn't analyzed yet @@ -77,7 +76,7 @@ A lattice for escape information, which holds the following properties: * `n` : through argument N There are utility constructors to create common `EscapeLattice`s, e.g., -- `NoEscape()`: the bottom element of this lattice, meaning it won't escape to anywhere +- `NoEscape()`: the bottom(-like) element of this lattice, meaning it won't escape to anywhere - `AllEscape()`: the topmost element of this lattice, meaning it will escape to everywhere `analyze_escapes` will transition these elements from the bottom to the top, @@ -90,17 +89,15 @@ An abstract state will be initialized with the bottom(-like) elements: """ struct EscapeLattice Analyzed::Bool - ReturnEscape::Bool - ThrownEscape::Bool - EscapeSites::BitSet + ReturnEscape::BitSet + ThrownEscape::BitSet AliasEscapes #::Union{FieldEscapes,ArrayEscapes,Bool} # TODO: ArgEscape::Int function EscapeLattice( Analyzed::Bool, - ReturnEscape::Bool, - ThrownEscape::Bool, - EscapeSites::BitSet, + ReturnEscape::BitSet, + ThrownEscape::BitSet, AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=#, ) @nospecialize AliasEscapes @@ -108,7 +105,6 @@ struct EscapeLattice Analyzed, ReturnEscape, ThrownEscape, - EscapeSites, AliasEscapes, ) end @@ -118,64 +114,76 @@ struct EscapeLattice # in order to avoid allocating non-concrete `NamedTuple`s AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=# = x.AliasEscapes; Analyzed::Bool = x.Analyzed, - ReturnEscape::Bool = x.ReturnEscape, - ThrownEscape::Bool = x.ThrownEscape, - EscapeSites::BitSet = x.EscapeSites, + ReturnEscape::BitSet = x.ReturnEscape, + ThrownEscape::BitSet = x.ThrownEscape, ) @nospecialize AliasEscapes return new( Analyzed, ReturnEscape, ThrownEscape, - EscapeSites, AliasEscapes, ) end end # precomputed default values in order to eliminate computations at each callsite -const BOT_ESCAPE_SITES = BitSet() -const ARGUMENT_ESCAPE_SITES = BitSet(0) -const TOP_ESCAPE_SITES = BitSet(0:100_000) +const BOT_RETURN_ESCAPE = BitSet() +const ARG_RETURN_ESCAPE = BitSet(0) +const TOP_RETURN_ESCAPE = BitSet(0:100_000) + +const BOT_THROWN_ESCAPE = BitSet() +const TOP_THROWN_ESCAPE = BitSet(0:100_000) const BOT_ALIAS_ESCAPES = false const TOP_ALIAS_ESCAPES = true # the constructors -NotAnalyzed() = EscapeLattice(false, false, false, BOT_ESCAPE_SITES, BOT_ALIAS_ESCAPES) # not formally part of the lattice -NoEscape() = EscapeLattice(true, false, false, BOT_ESCAPE_SITES, BOT_ALIAS_ESCAPES) -ReturnEscape(pc::Int) = EscapeLattice(true, true, false, BitSet(pc), BOT_ALIAS_ESCAPES) -ThrownEscape(pc::Int) = EscapeLattice(true, false, true, BitSet(pc), BOT_ALIAS_ESCAPES) -ArgumentReturnEscape() = EscapeLattice(true, true, false, ARGUMENT_ESCAPE_SITES, TOP_ALIAS_ESCAPES) # TODO allow interprocedural field analysis? -AllEscape() = EscapeLattice(true, true, true, TOP_ESCAPE_SITES, TOP_ALIAS_ESCAPES) +NotAnalyzed() = EscapeLattice(false, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) # not formally part of the lattice +NoEscape() = EscapeLattice(true, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) +ReturnEscape(pc::Int) = EscapeLattice(true, BitSet(pc), BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) +ArgumentReturnEscape() = EscapeLattice(true, ARG_RETURN_ESCAPE, BOT_THROWN_ESCAPE, TOP_ALIAS_ESCAPES) # TODO allow interprocedural field analysis? +AllReturnEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) +ThrownEscape(pc::Int) = EscapeLattice(true, BOT_RETURN_ESCAPE, BitSet(pc), BOT_ALIAS_ESCAPES) +ThrownEscape(pcs::BitSet) = EscapeLattice(true, BOT_RETURN_ESCAPE, pcs, BOT_ALIAS_ESCAPES) +AllEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, TOP_THROWN_ESCAPE, TOP_ALIAS_ESCAPES) + +const ⊥, ⊤ = NotAnalyzed(), AllEscape() # Convenience names for some ⊑ queries -has_not_analyzed(x::EscapeLattice) = x == NotAnalyzed() has_no_escape(x::EscapeLattice) = ignore_aliasescapes(x) ⊑ NoEscape() -has_return_escape(x::EscapeLattice) = x.ReturnEscape -has_return_escape(x::EscapeLattice, pc::Int) = has_return_escape(x) && pc in x.EscapeSites -has_thrown_escape(x::EscapeLattice) = x.ThrownEscape -has_thrown_escape(x::EscapeLattice, pc::Int) = has_thrown_escape(x) && pc in x.EscapeSites -has_all_escape(x::EscapeLattice) = AllEscape() ⊑ x - +has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) +has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape +has_thrown_escape(x::EscapeLattice) = !isempty(x.ThrownEscape) +has_thrown_escape(x::EscapeLattice, pc::Int) = pc in x.ThrownEscape +has_all_escape(x::EscapeLattice) = ⊤ ⊑ x + +# utility lattice constructors +ignore_thrownescapes(x::EscapeLattice) = EscapeLattice(x; ThrownEscape=BOT_THROWN_ESCAPE) ignore_aliasescapes(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_ESCAPES) -has_aliasescapes(x::EscapeLattice) = !isa(x.AliasEscapes, Bool) - -# TODO is_sroa_eligible: consider throwness? """ - is_sroa_eligible(x::EscapeLattice) -> Bool + is_load_forwardable(x::EscapeLattice) -> Bool -Queries allocation eliminability by SROA. +Queries if `x` is elibigle for store-to-load forwarding optimization. """ -function is_sroa_eligible(x::EscapeLattice) +function is_load_forwardable(x::EscapeLattice) if x.AliasEscapes === false || # allows this query to work for immutables since we don't impose escape on them isa(x.AliasEscapes, FieldEscapes) - return !has_return_escape(x) # TODO technically we also need to check !has_thrown_escape(x) as well + # NOTE technically we also need to check `!has_thrown_escape(x)` here as well, + # but we can also do equivalent check during forwarding + return true end return false end +""" + is_sroa_eligible(x::EscapeLattice) -> Bool + +Queries allocation eliminability by SROA. +""" +is_sroa_eligible(x::EscapeLattice) = is_load_forwardable(x) && !has_return_escape(x) + """ can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool @@ -190,6 +198,24 @@ can_elide_finalizer(x::EscapeLattice, pc::Int) = # we need to make sure this `==` operator corresponds to lattice equality rather than object equality, # otherwise `propagate_changes` can't detect the convergence x::EscapeLattice == y::EscapeLattice = begin + # fast pass: better to avoid top comparison + x === y && return true + xr, yr = x.ReturnEscape, y.ReturnEscape + if xr === TOP_RETURN_ESCAPE + yr === TOP_RETURN_ESCAPE || return false + elseif yr === TOP_RETURN_ESCAPE + return false # x.ReturnEscape === TOP_RETURN_ESCAPE + else + xr == yr || return false + end + xt, yt = x.ThrownEscape, y.ThrownEscape + if xt === TOP_THROWN_ESCAPE + yt === TOP_THROWN_ESCAPE || return false + elseif yt === TOP_THROWN_ESCAPE + return false # x.ThrownEscape === TOP_THROWN_ESCAPE + else + xt == yt || return false + end xf, yf = x.AliasEscapes, y.AliasEscapes if isa(xf, Bool) xf === yf || return false @@ -201,11 +227,7 @@ x::EscapeLattice == y::EscapeLattice = begin isa(yf, ArrayEscapes) || return false xf == yf || return false end - return x.Analyzed === y.Analyzed && - x.ReturnEscape === y.ReturnEscape && - x.ThrownEscape === y.ThrownEscape && - x.EscapeSites == y.EscapeSites && - true + return x.Analyzed === y.Analyzed end """ @@ -214,6 +236,28 @@ end The non-strict partial order over `EscapeLattice`. """ x::EscapeLattice ⊑ y::EscapeLattice = begin + # fast pass: better to avoid top comparison + if y === ⊤ + return true + elseif x === ⊤ + return false # return y === ⊤ + elseif x === ⊥ + return true + elseif y === ⊥ + return false # return x === ⊥ + end + xr, yr = x.ReturnEscape, y.ReturnEscape + if xr === TOP_RETURN_ESCAPE + yr !== TOP_RETURN_ESCAPE && return false + elseif yr !== TOP_RETURN_ESCAPE + xr ⊆ yr || return false + end + xt, yt = x.ThrownEscape, y.ThrownEscape + if xt === TOP_THROWN_ESCAPE + yt !== TOP_THROWN_ESCAPE && return false + elseif yt !== TOP_THROWN_ESCAPE + xt ⊆ yt || return false + end xf, yf = x.AliasEscapes, y.AliasEscapes if isa(xf, Bool) xf && yf !== true && return false @@ -235,14 +279,7 @@ x::EscapeLattice ⊑ y::EscapeLattice = begin yf === true || return false end end - if x.Analyzed ≤ y.Analyzed && - x.ReturnEscape ≤ y.ReturnEscape && - x.ThrownEscape ≤ y.ThrownEscape && - x.EscapeSites ⊆ y.EscapeSites && - true - return true - end - return false + return x.Analyzed ≤ y.Analyzed end """ @@ -267,6 +304,34 @@ x::EscapeLattice ⋤ y::EscapeLattice = !(y ⊑ x) Computes the join of `x` and `y` in the partial order defined by `EscapeLattice`. """ x::EscapeLattice ⊔ y::EscapeLattice = begin + # fast pass: better to avoid top join + if x === ⊤ || y === ⊤ + return ⊤ + elseif x === ⊥ + return y + elseif y === ⊥ + return x + end + xr, yr = x.ReturnEscape, y.ReturnEscape + if xr === TOP_RETURN_ESCAPE || yr === TOP_RETURN_ESCAPE + ReturnEscape = TOP_RETURN_ESCAPE + elseif xr === BOT_RETURN_ESCAPE + ReturnEscape = yr + elseif yr === BOT_RETURN_ESCAPE + ReturnEscape = xr + else + ReturnEscape = xr ∪ yr + end + xt, yt = x.ThrownEscape, y.ThrownEscape + if xt === TOP_THROWN_ESCAPE || yt === TOP_THROWN_ESCAPE + ThrownEscape = TOP_THROWN_ESCAPE + elseif xt === BOT_THROWN_ESCAPE + ThrownEscape = yt + elseif yt === BOT_THROWN_ESCAPE + ThrownEscape = xt + else + ThrownEscape = xt ∪ yt + end xf, yf = x.AliasEscapes, y.AliasEscapes if xf === true || yf === true AliasEscapes = true @@ -297,41 +362,14 @@ x::EscapeLattice ⊔ y::EscapeLattice = begin AliasEscapes = true # handle conflicting case conservatively end end - # try to avoid new allocations as minor optimizations - xe, ye = x.EscapeSites, y.EscapeSites - if xe === TOP_ESCAPE_SITES || ye === TOP_ESCAPE_SITES - EscapeSites = TOP_ESCAPE_SITES - elseif xe === BOT_ESCAPE_SITES - EscapeSites = ye - elseif ye === BOT_ESCAPE_SITES - EscapeSites = xe - else - EscapeSites = xe ∪ ye - end return EscapeLattice( x.Analyzed | y.Analyzed, - x.ReturnEscape | y.ReturnEscape, - x.ThrownEscape | y.ThrownEscape, - EscapeSites, + ReturnEscape, + ThrownEscape, AliasEscapes, ) end -""" - x::EscapeLattice ⊓ y::EscapeLattice -> EscapeLattice - -Computes the meet of `x` and `y` in the partial order defined by `EscapeLattice`. -""" -x::EscapeLattice ⊓ y::EscapeLattice = begin - return EscapeLattice( - x.Analyzed & y.Analyzed, - x.ReturnEscape & y.ReturnEscape, - x.ThrownEscape & y.ThrownEscape, - x.EscapeSites ∩ y.EscapeSites, - x.AliasEscapes, # FIXME - ) -end - # TODO setup a more effient struct for cache # which can discard escape information on SSS values and arguments that don't join dispatch signature @@ -340,11 +378,8 @@ const AliasSet = IntDisjointSet{Int} """ estate::EscapeState -Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`: -- `estate.arguments::Vector{EscapeLattice}`: escape information about "arguments"; - note that "argument" can include both call arguments and slots appearing in analysis frame -- `ssavalues::Vector{EscapeLattice}`: escape information about each SSA value -- `aliaset::IntDisjointSet{Int}`: a disjoint set that maintains aliased arguments and SSA values +Extended lattice that maps arguments and SSA values to escape information represented as `EscapeLattice`. +Escape information imposed on SSA IR element `x` can be retrieved by `estate[x]`. """ struct EscapeState escapes::Vector{EscapeLattice} @@ -353,7 +388,7 @@ struct EscapeState end function EscapeState(nargs::Int, nstmts::Int) escapes = EscapeLattice[ - 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : NotAnalyzed() for i in 1:(nargs+nstmts)] + 1 ≤ i ≤ nargs ? ArgumentReturnEscape() : ⊥ for i in 1:(nargs+nstmts)] aliaset = AliasSet(nargs+nstmts) return EscapeState(escapes, aliaset, nargs) end @@ -425,18 +460,65 @@ function getaliases(xidx::Int, estate::EscapeState) end end +""" + EscapeLatticeCache(x::EscapeLattice) -> x′::EscapeLatticeCache + +The data structure for caching `x::EscapeLattice` for interprocedural propagation, +which is slightly more efficient than the original `x` object. +""" +struct EscapeLatticeCache + AllEscape::Bool + ReturnEscape::Bool + ThrownEscape::Bool + function EscapeLatticeCache(x::EscapeLattice) + x === ⊤ && return new(true, true, true) + ReturnEscape = x.ReturnEscape === ARG_RETURN_ESCAPE ? false : true + ThrownEscape = isempty(x.ThrownEscape) ? false : true + return new(false, ReturnEscape, ThrownEscape) + end +end + +""" + cache_escapes!(linfo::MethodInstance, estate::EscapeState, _::IRCode) + +Transforms escape information of `estate` for interprocedural propagation, +and caches it in a global cache that can then be looked up later when +`linfo` callsite is seen again. +""" +function cache_escapes! end + +# when working outside of Core.Compiler, cache as much as information for later inspection and debugging if _TOP_MOD !== Core.Compiler struct EscapeCache - state::EscapeState - ir::IRCode # we preserve `IRCode` as well just for debugging purpose + cache::Vector{EscapeLatticeCache} + state::EscapeState # preserved just for debugging purpose + ir::IRCode # preserved just for debugging purpose end const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,EscapeCache}() - argescapes_from_cache(cache::EscapeCache) = - cache.state.escapes[1:cache.state.nargs] + function cache_escapes!(linfo::MethodInstance, estate::EscapeState, cacheir::IRCode) + cache = EscapeCache(to_interprocedural(estate), estate, cacheir) + GLOBAL_ESCAPE_CACHE[linfo] = cache + return cache + end + argescapes_from_cache(cache::EscapeCache) = cache.cache else - const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Vector{EscapeLattice}}() - argescapes_from_cache(cache::Vector{EscapeLattice}) = cache + const GLOBAL_ESCAPE_CACHE = IdDict{MethodInstance,Vector{EscapeLatticeCache}}() + function cache_escapes!(linfo::MethodInstance, estate::EscapeState, _::IRCode) + cache = to_interprocedural(estate) + GLOBAL_ESCAPE_CACHE[linfo] = cache + return cache + end + argescapes_from_cache(cache::Vector{EscapeLatticeCache}) = cache +end + +function to_interprocedural(estate::EscapeState) + cache = Vector{EscapeLatticeCache}(undef, estate.nargs) + for i = 1:estate.nargs + cache[i] = EscapeLatticeCache(estate.escapes[i]) + end + return cache end + __clear_escape_cache!() = empty!(GLOBAL_ESCAPE_CACHE) const EscapeChange = Pair{Int,EscapeLattice} @@ -450,7 +532,7 @@ struct AnalysisState end """ - analyze_escapes(ir::IRCode, nargs::Int) -> EscapeState + analyze_escapes(ir::IRCode, nargs::Int) -> estate::EscapeState Analyzes escape information in `ir`. `nargs` is the number of actual arguments of the analyzed call. @@ -462,6 +544,7 @@ function analyze_escapes(ir::IRCode, nargs::Int) # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties estate = EscapeState(nargs, nstmts) changes = Changes() # stashes changes that happen at current statement + tryregions = compute_tryregions(ir) astate = AnalysisState(ir, estate, changes) local debug_itr_counter = 0 @@ -483,7 +566,7 @@ function analyze_escapes(ir::IRCode, nargs::Int) elseif head === :(=) lhs, rhs = stmt.args if isa(lhs, GlobalRef) # global store - add_escape_change!(astate, rhs, AllEscape()) + add_escape_change!(astate, rhs, ⊤) else invalid_escape_assignment!(ir, pc) end @@ -494,35 +577,21 @@ function analyze_escapes(ir::IRCode, nargs::Int) elseif is_meta_expr_head(head) # meta expressions doesn't account for any usages continue - elseif head === :static_parameter - # :static_parameter refers any of static parameters, but since they exist - # statically, we're really not interested in their escapes - continue - elseif head === :copyast - # copyast simply copies a surface syntax AST, and should never use any of arguments or SSA values - continue - elseif head === :undefcheck - # undefcheck is temporarily inserted by compiler - # it will be processd be later pass so it won't change any of escape states + elseif head === :enter || head === :leave || head === :the_exception || head === :pop_exception + # ignore these expressions since escapes via exceptions are handled by `escape_exception!` + # `escape_exception!` conservatively propagates `AllEscape` anyway, + # and so escape information imposed on `:the_exception` isn't computed continue - elseif head === :the_exception - # we don't propagate escape information on exceptions via this expression, but rather - # use a dedicated lattice property `ThrownEscape` - continue - elseif head === :isdefined - # just returns `Bool`, nothing accounts for any usages - continue - elseif head === :enter || head === :leave || head === :pop_exception - # these exception frame managements doesn't account for any usages - # we can just ignore escape information from - continue - elseif head === :gc_preserve_begin || head === :gc_preserve_end - # `GC.@preserve` may "use" arbitrary values, but we can just ignore the escape information - # imposed on `GC.@preserve` expressions since they're supposed to never be used elsewhere + elseif head === :static_parameter || # this exists statically, not interested in its escape + head === :copyast || # XXX can this account for some escapes? + head === :undefcheck || # XXX can this account for some escapes? + head === :isdefined || # just returns `Bool`, nothing accounts for any escapes + head === :gc_preserve_begin || # `GC.@preserve` expressions themselves won't be used anywhere + head === :gc_preserve_end # `GC.@preserve` expressions themselves won't be used anywhere continue else for x in stmt.args - add_escape_change!(astate, x, AllEscape()) + add_escape_change!(astate, x, ⊤) end end elseif isa(stmt, ReturnNode) @@ -538,7 +607,7 @@ function analyze_escapes(ir::IRCode, nargs::Int) elseif isa(stmt, UpsilonNode) escape_val!(astate, pc, stmt) elseif isa(stmt, GlobalRef) # global load - add_escape_change!(astate, SSAValue(pc), AllEscape()) + add_escape_change!(astate, SSAValue(pc), ⊤) elseif isa(stmt, SSAValue) # after SROA, we may see SSA value as statement escape_ssa!(astate, pc, stmt) else @@ -553,6 +622,8 @@ function analyze_escapes(ir::IRCode, nargs::Int) empty!(changes) end + tryregions !== nothing && escape_exception!(astate, tryregions) + debug_itr_counter += 1 anyupdate || break @@ -609,6 +680,7 @@ function propagate_alias_change!(estate::EscapeState, change::AliasChange) end function add_escape_change!(astate::AnalysisState, @nospecialize(x), info::EscapeLattice) + info === ⊥ && return # performance optimization xidx = iridx(x, astate.estate) if xidx !== nothing if !isbitstype(widenconst(argextype(x, astate.ir))) @@ -662,13 +734,92 @@ escape_ssa!(astate::AnalysisState, pc::Int, ssa::SSAValue) = # add_escape_change!(astate, lhs, vinfo) # end +# linear scan to find regions in which potential throws will be caught +function compute_tryregions(ir::IRCode) + tryregions = nothing + for idx in 1:length(ir.stmts) + stmt = ir.stmts[idx][:inst] + if isexpr(stmt, :enter) + tryregions === nothing && (tryregions = UnitRange{Int}[]) + leave_block = stmt.args[1]::Int + leave_pc = first(ir.cfg.blocks[leave_block].stmts) + push!(tryregions, idx:leave_pc) + end + end + return tryregions +end + +""" + escape_exception!(astate::AnalysisState, tryregions::Vector{UnitRange{Int}}) + +Propagates escapes via exceptions that can happen in `tryregions`. + +Naively it seems enough to propagate escape information imposed on `:the_exception` object, +but actually there are several other ways to access to the exception object such as +`Base.current_exceptions` and manual catch of `rethrow`n object. +For example, escape analysis needs to account for potential escape of the allocated object +via `rethrow_escape!()` call in the example below: +```julia +const Gx = Ref{Any}() +@noinline function rethrow_escape!() + try + rethrow() + catch err + Gx[] = err + end +end +unsafeget(x) = isassigned(x) ? x[] : throw(x) + +code_escapes() do + r = Ref{String}() + try + t = unsafeget(r) + catch err + t = typeof(err) # `err` (which `r` may alias to) doesn't escape here + rethrow_escape!() # `r` can escape here + end + return t +end +``` + +As indicated by the above example, it requires a global analysis in addition to a base escape +analysis to reason about all possible escapes via existing exception interfaces correctly. +For now we conservatively always propagate `AllEscape` to all potentially thrown objects, +since such an additional analysis might not be worthwhile to do given that exception handlings +and error paths usually don't need to be very performance sensitive, and optimizations of +error paths might be very ineffective anyway since they are sometimes "unoptimized" +intentionally for latency reasons. +""" +function escape_exception!(astate::AnalysisState, tryregions::Vector{UnitRange{Int}}) + estate = astate.estate + # NOTE if `:the_exception` is the only way to access the exception, we can do: + # exc = SSAValue(pc) + # excinfo = estate[exc] + excinfo = ⊤ + escapes = estate.escapes + for i in 1:length(escapes) + x = escapes[i] + xt = x.ThrownEscape + xt === TOP_THROWN_ESCAPE && @goto propagate_exception_escape # fast pass + for pc in x.ThrownEscape + for region in tryregions + pc in region && @goto propagate_exception_escape # early break because of AllEscape + end + end + continue + @label propagate_exception_escape + xval = irval(i, estate) + add_escape_change!(astate, xval, excinfo) + end +end + function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) linfo = first(args)::MethodInstance cache = get(GLOBAL_ESCAPE_CACHE, linfo, nothing) if cache === nothing for i in 2:length(args) x = args[i] - add_escape_change!(astate, x, AllEscape()) + add_escape_change!(astate, x, ⊤) end else argescapes = argescapes_from_cache(cache) @@ -683,44 +834,41 @@ function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) argi = nargs end arginfo = argescapes[argi] - isempty(arginfo.ReturnEscape) && invalid_escape_invoke!(astate, linfo, linfo_estate) info = from_interprocedural(arginfo, retinfo, pc) add_escape_change!(astate, arg, info) end end end -# reinterpret the escape information imposed on the callee argument (`arginfo`) in the -# context of the caller frame using the escape information imposed on the return value (`retinfo`) -function from_interprocedural(arginfo::EscapeLattice, retinfo::EscapeLattice, pc::Int) - @assert arginfo.ReturnEscape - if arginfo.ThrownEscape - EscapeSites = BitSet(pc) - else - EscapeSites = BOT_ESCAPE_SITES - end +""" + from_interprocedural(arginfo::EscapeLatticeCache, retinfo::EscapeLattice, pc::Int) -> x::EscapeLattice + +Reinterprets the escape information imposed on the call argument which is cached as `arginfo` +in the context of the caller frame, where `retinfo` is the escape information imposed on +the return value and `pc` is the SSA statement number of the return value. +""" +function from_interprocedural(arginfo::EscapeLatticeCache, retinfo::EscapeLattice, pc::Int) + arginfo.AllEscape && return ⊤ + + ThrownEscape = arginfo.ThrownEscape ? BitSet(pc) : BOT_THROWN_ESCAPE + newarginfo = EscapeLattice( - #=Analyzed=#true, #=ReturnEscape=#false, arginfo.ThrownEscape, EscapeSites, - # FIXME implement interprocedural effect-analysis + #=Analyzed=#true, #=ReturnEscape=#BOT_RETURN_ESCAPE, ThrownEscape, + # FIXME implement interprocedural memory effect-analysis # currently, this essentially disables the entire field analysis # it might be okay from the SROA point of view, since we can't remove the allocation # as far as it's passed to a callee anyway, but still we may want some field analysis - # in order to stack allocate it + # for e.g. stack allocation or some other IPO optimizations TOP_ALIAS_ESCAPES) - if arginfo.EscapeSites === ARGUMENT_ESCAPE_SITES + + if !arginfo.ReturnEscape # if this is simply passed as the call argument, we can discard the `ReturnEscape` # information and just propagate the other escape information return newarginfo - else - # if this can be returned, we have to merge its escape information with - # that of the current statement - return newarginfo ⊔ retinfo end -end -@noinline function invalid_escape_invoke!(astate::AnalysisState, linfo::MethodInstance, linfo_estate::EscapeState) - @eval Main (astate = $astate; linfo = $linfo; linfo_estate = $linfo_estate) - error("invalid escape lattice element returned from inter-procedural context: inspect `Main.astate`, `Main.linfo` and `Main.linfo_estate`") + # if this argument can be "returned", we have to merge its escape information with that imposed on the return value + return newarginfo ⊔ retinfo end @noinline function invalid_escape_assignment!(ir::IRCode, pc::Int) @@ -731,9 +879,6 @@ end function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) obj = SSAValue(pc) objinfo = astate.estate[obj] - if objinfo == NotAnalyzed() - objinfo = NoEscape() - end AliasEscapes = objinfo.AliasEscapes nargs = length(args) if isa(AliasEscapes, Bool) @@ -836,7 +981,7 @@ function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) f = singleton_type(ft) if isa(f, Core.IntrinsicFunction) - # COMBAK we may break soundness and need to account for some aliasing here, e.g. `pointerref` + # XXX somehow `:call` expression can creep in here, ideally we should be able to do: # argtypes = Any[argextype(args[i], astate.ir) for i = 2:length(args)] argtypes = Any[] for i = 2:length(args) @@ -844,14 +989,14 @@ function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) push!(argtypes, isexpr(arg, :call) ? Any : argextype(arg, ir)) end intrinsic_nothrow(f, argtypes) || add_thrown_escapes!(astate, pc, args, 2) - return + return # TODO accounts for pointer operations end result = escape_builtin!(f, astate, pc, args) if result === missing # if this call hasn't been handled by any of pre-defined handlers, # we escape this call conservatively for i in 2:length(args) - add_escape_change!(astate, args[i], AllEscape()) + add_escape_change!(astate, args[i], ⊤) end return elseif result === true @@ -920,7 +1065,7 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg obj = args[2] typ = widenconst(argextype(obj, ir)) if hasintersect(typ, Module) # global load - add_escape_change!(astate, SSAValue(pc), AllEscape()) + add_escape_change!(astate, SSAValue(pc), ⊤) end if isa(obj, SSAValue) || isa(obj, Argument) objinfo = estate[obj] @@ -944,9 +1089,6 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg # imposed on the return value of this `getfield` call to the object itself # as the most conservative propagation ssainfo = estate[SSAValue(pc)] - if ssainfo == NotAnalyzed() - ssainfo = NoEscape() - end add_escape_change!(astate, obj, ssainfo) elseif isa(AliasEscapes, FieldEscapes) nfields = fieldcount_noerror(typ) @@ -1001,7 +1143,7 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar objinfo = estate[obj] else # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively - add_escape_change!(astate, val, AllEscape()) + add_escape_change!(astate, val, ⊤) @goto add_thrown_escapes end AliasEscapes = objinfo.AliasEscapes @@ -1066,9 +1208,6 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar end # also propagate escape information imposed on the return value of this `setfield!` ssainfo = estate[SSAValue(pc)] - if ssainfo == NotAnalyzed() - ssainfo = NoEscape() - end add_escape_change!(astate, val, ssainfo) # compute the throwness of this setfield! call here since builtin_nothrow doesn't account for that @label add_thrown_escapes @@ -1084,22 +1223,22 @@ const Arrayish = Union{Array,Core.ImmutableArray} function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 4 || return false - # check potential escapes from this arrayref call - # NOTE here we essentially only need to account for TypeError, assuming that - # UndefRefError or BoundsError don't capture any of the arguments here + # check potential thrown escapes from this arrayref call argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] boundcheckt = argtypes[1] aryt = argtypes[2] if !array_builtin_common_typecheck(Arrayish, boundcheckt, aryt, argtypes, 3) add_thrown_escapes!(astate, pc, args, 2) end + ary = args[3] + inbounds = isa(boundcheckt, Const) && !boundcheckt.val::Bool + inbounds || add_escape_change!(astate, ary, ThrownEscape(pc)) # we don't track precise index information about this array and thus don't know what values # can be referenced here: directly propagate the escape information imposed on the return # value of this `arrayref` call to the array itself as the most conservative propagation # but also with updated index information # TODO enable index analysis when constant values are available? estate = astate.estate - ary = args[3] if isa(ary, SSAValue) || isa(ary, Argument) aryinfo = estate[ary] else @@ -1115,25 +1254,13 @@ function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, arg end @label conservative_propagation ssainfo = estate[ret] - if ssainfo == NotAnalyzed() - ssainfo = NoEscape() - end add_escape_change!(astate, ary, ssainfo) - if isa(boundcheckt, Const) - if boundcheckt.val::Bool - add_escape_change!(astate, ary, ThrownEscape(pc)) - end - end elseif isa(AliasEscapes, ArrayEscapes) # record the return value of this `arrayref` call as a possibility that imposes escape AliasEscapes = copy(AliasEscapes) @label record_element_escape push!(AliasEscapes, iridx(ret, estate)) - if isa(boundcheckt, Const) # record possible BoundsError at this arrayref - if boundcheckt.val::Bool - push!(AliasEscapes, SSAValue(pc)) - end - end + inbounds || push!(AliasEscapes, SSAValue(pc)) # record possible BoundsError at this arrayref add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) else # this object has been used as struct, but it is used as array here (thus should throw) @@ -1158,18 +1285,20 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg arrayset_typecheck(aryt, valt)) add_thrown_escapes!(astate, pc, args, 2) end + ary = args[3] + val = args[4] + inbounds = isa(boundcheckt, Const) && !boundcheckt.val::Bool + inbounds || add_escape_change!(astate, ary, ThrownEscape(pc)) # we don't track precise index information about this array and won't record what value # is being assigned here: directly propagate the escape information of this array to # the value being assigned as the most conservative propagation # TODO enable index analysis when constant values are available? estate = astate.estate - ary = args[3] - val = args[4] if isa(ary, SSAValue) || isa(ary, Argument) aryinfo = estate[ary] else # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively - add_escape_change!(astate, val, AllEscape()) + add_escape_change!(astate, val, ⊤) return true end AliasEscapes = aryinfo.AliasEscapes @@ -1187,7 +1316,7 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg add_escape_change!(astate, val, estate[x]) add_alias_change!(astate, val, x) else - add_escape_change!(astate, val, ThrownEscape(xidx.id)) + add_escape_change!(astate, val, ThrownEscape((xidx::SSAValue).id)) end end @label add_ary_escape @@ -1201,9 +1330,6 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg end # also propagate escape information imposed on the return value of this `arrayset` ssainfo = estate[SSAValue(pc)] - if ssainfo == NotAnalyzed() - ssainfo = NoEscape() - end add_escape_change!(astate, ary, ssainfo) return true end @@ -1256,33 +1382,8 @@ function escape_array_resize!(boundserror::Bool, ninds::Int, indt ⊑ₜ Integer || return add_thrown_escapes!(astate, pc, args) end if boundserror - if isa(ary, SSAValue) || isa(ary, Argument) - estate = astate.estate - aryinfo = estate[ary] - AliasEscapes = aryinfo.AliasEscapes - if isa(AliasEscapes, Bool) - if !AliasEscapes - # the elements of this array haven't been analyzed yet: set ArrayEscapes now - AliasEscapes = ArrayEscapes() - @goto record_element_escape - end - @label conservative_propagation - # array resizing can potentially throw `BoundsError`, impose it now - add_escape_change!(astate, ary, ThrownEscape(pc)) - elseif isa(AliasEscapes, ArrayEscapes) - AliasEscapes = copy(AliasEscapes) - @label record_element_escape - # array resizing can potentially throw `BoundsError`, record it now - push!(AliasEscapes, SSAValue(pc)) - add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) - else - # this object has been used as struct, but it is used as array here (thus should throw) - # update ary's element information and just handle this case conservatively - @assert isa(AliasEscapes, FieldEscapes) - aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) - @goto conservative_propagation - end - end + # this array resizing can potentially throw `BoundsError`, impose it now + add_escape_change!(astate, ary, ThrownEscape(pc)) end end @@ -1337,13 +1438,13 @@ end # return true # end -escape_builtin!(::typeof(Core.arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = - escape_immutable_array!(Array, astate, pc, args) -escape_builtin!(::typeof(Core.mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = - escape_immutable_array!(Array, astate, pc, args) -escape_builtin!(::typeof(Core.arraythaw), astate::AnalysisState, pc::Int, args::Vector{Any}) = - escape_immutable_array!(Core.ImmutableArray, astate, pc, args) -function escape_immutable_array!(@nospecialize(arytype), astate::AnalysisState, pc::Int, args::Vector{Any}) +escape_builtin!(::typeof(arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = + is_safe_immutable_array_op(Array, astate, pc, args) +escape_builtin!(::typeof(mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = + is_safe_immutable_array_op(Array, astate, pc, args) +escape_builtin!(::typeof(arraythaw), astate::AnalysisState, pc::Int, args::Vector{Any}) = + is_safe_immutable_array_op(ImmutableArray, astate, pc, args) +function is_safe_immutable_array_op(@nospecialize(arytype), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) == 2 || return false argextype(args[2], astate.ir) ⊑ₜ arytype || return false return true @@ -1351,7 +1452,7 @@ end # NOTE define fancy package utilities when developing EA as an external package if _TOP_MOD !== Core.Compiler - include(@__MODULE__, "utils.jl") + include(@__MODULE__, "EAUtils.jl") end end # baremodule EscapeAnalysis diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 1082bb08a26ce..cf8cbc7e2195c 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -521,7 +521,7 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) isa(def, Method) ? Int(def.nargs) : 0 end estate = analyze_escapes(ir, nargs) - setindex!(GLOBAL_ESCAPE_CACHE, estate.escapes[1:estate.nargs], sv.linfo) + cache_escapes!(sv.linfo, estate, ir) @timeit "memory opt" ir = memory_opt!(ir, estate) if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index b945c9cda79a1..7773d0cbc4885 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1405,8 +1405,10 @@ function memory_opt!(ir::IRCode, estate) length(stmt.args) ≥ 2 || continue ary = stmt.args[2] if isa(ary, SSAValue) - # if array doesn't escape, we can just change the tag and avoid allocation - has_no_escape(estate[ary]) || continue + # we can change this arrayfreeze call (which incurs allocation) to mutating_arrayfreeze + # so that it just changes the type tag of the array and avoids the allocation + # as far as the array doesn't escape at this point (meaning we can ignore ThrownEscape here) + has_return_escape(estate[ary]) && continue stmt.args[1] = GlobalRef(Core, :mutating_arrayfreeze) end end From eaf50058dafcaedd5ba2161524f2b816a0a46f91 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Fri, 14 Jan 2022 05:54:55 +0900 Subject: [PATCH 33/41] no broken tests now --- test/compiler/immutablearray.jl | 107 +++++++++++++++----------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index bb6a837880df4..da3d327144370 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -131,7 +131,7 @@ let # arrayset @test allocated == @allocated optimizable1(ImmutableArray) end - function optimizable2(gen) + function unoptimizable(gen) a = Matrix{Float64}(undef, 5, 2) for i = 1:5 for j = 1:2 @@ -140,14 +140,14 @@ let # arrayset end return gen(a) end - let src = code_typed1(optimizable2, (Type{ImmutableArray},)) + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) @test count(is_array_alloc, src.code) == 1 @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 @test count(iscall((src, arrayfreeze)), src.code) == 0 - optimizable2(identity) - allocated = @allocated optimizable2(identity) - optimizable2(ImmutableArray) - @test allocated == @allocated optimizable2(ImmutableArray) + unoptimizable(identity) + allocated = @allocated unoptimizable(identity) + unoptimizable(ImmutableArray) + @test allocated == @allocated unoptimizable(ImmutableArray) end end @@ -219,12 +219,12 @@ let # ignore ThrownEscape if it never happens when `arrayfreeze` is called end let src = code_typed1(optimizable, (Type{ImmutableArray},Int,)) @test count(is_array_alloc, src.code) == 1 - @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 - @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 optimizable(identity, 42) allocated = @allocated optimizable(identity, 42) optimizable(ImmutableArray, 42) - @test_broken allocated == @allocated optimizable(ImmutableArray, 42) + @test allocated == @allocated optimizable(ImmutableArray, 42) end end @noinline function ipo_getindex′(a, n) @@ -240,12 +240,12 @@ let # ignore ThrownEscape if it never happens when `arrayfreeze` is called (inte let src = code_typed1(optimizable, (Type{ImmutableArray},)) @test count(is_array_alloc, src.code) == 1 @test count(isinvoke(:ipo_getindex′), src.code) == 1 - @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 - @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 optimizable(identity) allocated = @allocated optimizable(identity) optimizable(ImmutableArray) - @test_broken allocated == @allocated optimizable(ImmutableArray) + @test allocated == @allocated optimizable(ImmutableArray) end end @@ -277,53 +277,31 @@ function optimizable_aa(gen, n) # can't be a closure somehow end let src = code_typed1(optimizable_aa, (Type{ImmutableArray},Int)) @test count(is_array_alloc, src.code) == 1 - @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 - @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 0 optimizable_aa(identity, 100) allocated = @allocated optimizable_aa(identity, 100) optimizable_aa(ImmutableArray, 100) - @test_broken allocated == @allocated optimizable_aa(ImmutableArray, 100) + @test allocated == @allocated optimizable_aa(ImmutableArray, 100) end -const g = Ref{Any}() -let # BoundsError (assumes BoundsError doesn't capture arrays) - function optimizable1(gen) +let # should be possible if we change BoundsError semantics (so that it doesn't capture the indexed array) + function optimizable(gen) a = [1,2,3] try getindex(a, 4) catch - return gen(a) - end - end - let src = code_typed1(optimizable1, (Type{ImmutableArray},)) - @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, arrayfreeze)), src.code) == 0 - optimizable1(identity) - allocated = @allocated optimizable1(identity) - optimizable1(ImmutableArray) - @test allocated == @allocated optimizable1(ImmutableArray) - end - - function optimizable2(gen) - a = [1,2,3] - try - getindex(a, 4) - catch e - g[] = e.a # XXX these tests pass, but this optimization is actually incorrect until BoundsError doesn't escape its objects - return gen(a) end + return gen(a) end - let src = code_typed1(optimizable2, (Type{ImmutableArray},)) + let src = code_typed1(optimizable, (Type{ImmutableArray},)) @test count(is_array_alloc, src.code) == 1 - @test count(iscall((src, mutating_arrayfreeze)), src.code) == 1 - @test count(iscall((src, arrayfreeze)), src.code) == 0 - optimizable2(identity) - allocated = @allocated optimizable2(identity) - optimizable2(ImmutableArray) - local ia - @test allocated == @allocated ia = optimizable2(ImmutableArray) - @test_broken g[] !== ia + @test_broken count(iscall((src, mutating_arrayfreeze)), src.code) == 1 + @test_broken count(iscall((src, arrayfreeze)), src.code) == 0 + optimizable(identity) + allocated = @allocated optimizable(identity) + optimizable(ImmutableArray) + @test_broken allocated == @allocated optimizable(ImmutableArray) end end @@ -341,11 +319,8 @@ let # return escape @test count(is_array_alloc, src.code) == 1 @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 @test count(iscall((src, arrayfreeze)), src.code) == 1 - unoptimizable(identity) - allocated = @allocated unoptimizable(identity) unoptimizable(ImmutableArray) - local a, b - @test allocated < @allocated a, b = unoptimizable(ImmutableArray) + a, b = unoptimizable(ImmutableArray) @test a !== b @test !(a isa ImmutableArray) end @@ -376,10 +351,8 @@ let # global escape @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 @test count(iscall((src, arrayfreeze)), src.code) == 1 unoptimizable(identity) - allocated = @allocated unoptimizable(identity) unoptimizable(ImmutableArray) - local a - @test allocated < @allocated a = unoptimizable(ImmutableArray) + a = unoptimizable(ImmutableArray) @test global_array !== a @test !(global_array isa ImmutableArray) end @@ -396,10 +369,8 @@ let # global escape @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 @test count(iscall((src, arrayfreeze)), src.code) == 1 unoptimizable(identity) - allocated = @allocated unoptimizable(identity) unoptimizable(ImmutableArray) - local a - @test allocated < @allocated a = unoptimizable(ImmutableArray) + a = unoptimizable(ImmutableArray) @test Rx[] !== a @test !(Rx[] isa ImmutableArray) end @@ -429,6 +400,28 @@ let # escapes via exception end end +const g = Ref{Any}() +let # escapes via BoundsError + function unoptimizable(gen) + a = [1,2,3] + try + getindex(a, 4) + catch e + g[] = e.a + end + return gen(a) + end + let src = code_typed1(unoptimizable, (Type{ImmutableArray},)) + @test count(is_array_alloc, src.code) == 1 + @test count(iscall((src, arrayfreeze)), src.code) == 1 + @test count(iscall((src, mutating_arrayfreeze)), src.code) == 0 + unoptimizable(identity) + unoptimizable(ImmutableArray) + ia = unoptimizable(ImmutableArray) + @test g[] !== ia + end +end + # @testset "maybecopy tests" begin # g = nothing # global From 8110fadb739b252abb18483261fd30423b2eefe8 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 15 Jan 2022 12:41:34 +0900 Subject: [PATCH 34/41] update EA --- base/compiler/EscapeAnalysis/EAUtils.jl | 8 - .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 143 ++++++++++-------- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EAUtils.jl b/base/compiler/EscapeAnalysis/EAUtils.jl index 295fdeac813bd..78ebf5eff206f 100644 --- a/base/compiler/EscapeAnalysis/EAUtils.jl +++ b/base/compiler/EscapeAnalysis/EAUtils.jl @@ -403,11 +403,3 @@ function print_with_info(preprint, postprint, io::IO, ir::IRCode) end end # module EAUtils - -if EA_AS_PKG - using .EAUtils: code_escapes, @code_escapes - export code_escapes, @code_escapes -else - using .EAUtils: code_escapes - export code_escapes -end diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 3b534f18883e9..1e63c2e07e366 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -27,7 +27,7 @@ import ._TOP_MOD: # Base definitions @__MODULE__, @eval, @assert, @nospecialize, @inbounds, @inline, @noinline, @label, @goto, !, !==, !=, ≠, +, -, ≤, <, ≥, >, &, |, include, error, missing, copy, Vector, BitSet, IdDict, IdSet, UnitRange, ∪, ⊆, ∩, :, ∈, ∉, in, length, get, first, last, - isempty, isassigned, pop!, push!, empty!, max, min, Csize_t + isempty, isassigned, pop!, push!, pushfirst!, empty!, max, min, Csize_t import Core.Compiler: # Core.Compiler specific definitions isbitstype, isexpr, is_meta_expr_head, println, IRCode, IR_FLAG_EFFECT_FREE, widenconst, argextype, singleton_type, fieldcount_noerror, @@ -43,10 +43,7 @@ end # XXX better to be IdSet{Int}? const FieldEscape = BitSet const FieldEscapes = Vector{BitSet} -# for x in ArrayEscapes: -# - x::Int: `irval(x, estate)` imposes escapes on the array elements -# - x::SSAValue: array elements can potentially escape via BoundsError at this SSA statement -const ArrayEscapes = IdSet{Union{Int,SSAValue}} +const ArrayEscapes = IdSet{Int} """ x::EscapeLattice @@ -59,16 +56,13 @@ A lattice for escape information, which holds the following properties: - `x.ThrownEscape::BitSet`: records SSA statements where `x` can be thrown as exception: this information will be used by `escape_exception!` to propagate potential escapes via exception - `x.AliasEscapes::Union{FieldEscapes,ArrayEscapes,Bool}`: maintains all possible values - that may escape objects that can be referenced from `x`: + that can be aliased to fields or array elements of `x`: * `x.AliasEscapes === false` indicates the fields/elements of `x` isn't analyzed yet * `x.AliasEscapes === true` indicates the fields/elements of `x` can't be analyzed, e.g. the type of `x` is not known or is not concrete and thus its fields/elements can't be known precisely - * `x.AliasEscapes::FieldEscapes` records all the possible values that can escape fields of `x`, - which allows EA to propagate propagate escape information imposed on a field - of `x` to its values (by analyzing `Expr(:new, ...)` and `setfield!(x, ...)`). - * `x.AliasEscapes::ArrayEscapes` records all the possible values that can escape elements of `x`, - or all SSA staements that can potentially escape elements of `x` via `BoundsError`. + * `x.AliasEscapes::FieldEscapes` records all the possible values that can be aliased fields of object `x`, + * `x.AliasEscapes::ArrayEscapes` records all the possible values that be aliased to elements of array `x` - `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through `setfield!` on argument(s) * `-1` : no escape @@ -531,6 +525,15 @@ struct AnalysisState changes::Changes end +function getinst(ir::IRCode, idx::Int) + nstmts = length(ir.stmts) + if idx ≤ nstmts + return ir.stmts[idx] + else + return ir.new_nodes.stmts[idx - nstmts] + end +end + """ analyze_escapes(ir::IRCode, nargs::Int) -> estate::EscapeState @@ -539,7 +542,7 @@ Analyzes escape information in `ir`. """ function analyze_escapes(ir::IRCode, nargs::Int) stmts = ir.stmts - nstmts = length(stmts) + nstmts = length(stmts) + length(ir.new_nodes.stmts) # only manage a single state, some flow-sensitivity is encoded as `EscapeLattice` properties estate = EscapeState(nargs, nstmts) @@ -552,7 +555,7 @@ function analyze_escapes(ir::IRCode, nargs::Int) local anyupdate = false for pc in nstmts:-1:1 - stmt = stmts[pc][:inst] + stmt = getinst(ir, pc)[:inst] # collect escape information if isa(stmt, Expr) @@ -568,7 +571,7 @@ function analyze_escapes(ir::IRCode, nargs::Int) if isa(lhs, GlobalRef) # global store add_escape_change!(astate, rhs, ⊤) else - invalid_escape_assignment!(ir, pc) + unexpected_assignment!(ir, pc) end elseif head === :foreigncall escape_foreigncall!(astate, pc, stmt.args) @@ -601,17 +604,18 @@ function analyze_escapes(ir::IRCode, nargs::Int) elseif isa(stmt, PhiNode) escape_edges!(astate, pc, stmt.values) elseif isa(stmt, PiNode) - escape_val!(astate, pc, stmt) + escape_val_ifdefined!(astate, pc, stmt) elseif isa(stmt, PhiCNode) escape_edges!(astate, pc, stmt.values) elseif isa(stmt, UpsilonNode) - escape_val!(astate, pc, stmt) + escape_val_ifdefined!(astate, pc, stmt) elseif isa(stmt, GlobalRef) # global load add_escape_change!(astate, SSAValue(pc), ⊤) - elseif isa(stmt, SSAValue) # after SROA, we may see SSA value as statement - escape_ssa!(astate, pc, stmt) - else - @assert stmt isa GotoNode || stmt isa GotoIfNot || stmt === nothing # TODO remove me + elseif isa(stmt, SSAValue) + escape_val!(astate, pc, stmt) + elseif isa(stmt, Argument) + escape_val!(astate, pc, stmt) + else # otherwise `stmt` can be GotoNode, GotoIfNot, and inlined values etc. continue end @@ -680,47 +684,54 @@ function propagate_alias_change!(estate::EscapeState, change::AliasChange) end function add_escape_change!(astate::AnalysisState, @nospecialize(x), info::EscapeLattice) - info === ⊥ && return # performance optimization + info === ⊥ && return nothing # performance optimization xidx = iridx(x, astate.estate) if xidx !== nothing if !isbitstype(widenconst(argextype(x, astate.ir))) push!(astate.changes, EscapeChange(xidx, info)) end end + return nothing end function add_alias_change!(astate::AnalysisState, @nospecialize(x), @nospecialize(y)) - xidx = iridx(x, astate.estate) - yidx = iridx(y, astate.estate) - if xidx !== nothing && yidx !== nothing - push!(astate.changes, AliasChange(xidx, yidx)) + if isa(x, GlobalRef) + return add_escape_change!(astate, y, ⊤) + elseif isa(y, GlobalRef) + return add_escape_change!(astate, x, ⊤) end + estate = astate.estate + xidx = iridx(x, estate) + yidx = iridx(y, estate) + if xidx !== nothing && yidx !== nothing + pushfirst!(astate.changes, AliasChange(xidx, yidx)) # propagate `AliasChange` first for faster convergence + xinfo = estate.escapes[xidx] + yinfo = estate.escapes[yidx] + xyinfo = xinfo ⊔ yinfo + add_escape_change!(astate, x, xyinfo) + add_escape_change!(astate, y, xyinfo) + end + return nothing end function escape_edges!(astate::AnalysisState, pc::Int, edges::Vector{Any}) - info = astate.estate[SSAValue(pc)] + ret = SSAValue(pc) for i in 1:length(edges) if isassigned(edges, i) v = edges[i] - add_escape_change!(astate, v, info) - add_alias_change!(astate, SSAValue(pc), v) + add_alias_change!(astate, ret, v) end end end -escape_val!(astate::AnalysisState, pc::Int, x) = - isdefined(x, :val) && _escape_val!(astate, pc, x.val) +escape_val_ifdefined!(astate::AnalysisState, pc::Int, x) = + isdefined(x, :val) && escape_val!(astate, pc, x.val) -function _escape_val!(astate::AnalysisState, pc::Int, @nospecialize(val)) +function escape_val!(astate::AnalysisState, pc::Int, @nospecialize(val)) ret = SSAValue(pc) - info = astate.estate[ret] - add_escape_change!(astate, val, info) add_alias_change!(astate, ret, val) end -escape_ssa!(astate::AnalysisState, pc::Int, ssa::SSAValue) = - _escape_val!(astate, pc, ssa) - # NOTE if we don't maintain the alias set that is separated from the lattice state, we can do # soemthing like below: it essentially incorporates forward escape propagation in our default # backward propagation, and leads to inefficient convergence that requires more iterations @@ -746,6 +757,10 @@ function compute_tryregions(ir::IRCode) push!(tryregions, idx:leave_pc) end end + for idx in 1:length(ir.new_nodes.stmts) + stmt = ir.new_nodes.stmts[idx][:inst] + @assert !isexpr(stmt, :enter) "try/catch inside new_nodes unsupported" + end return tryregions end @@ -859,7 +874,7 @@ function from_interprocedural(arginfo::EscapeLatticeCache, retinfo::EscapeLattic # it might be okay from the SROA point of view, since we can't remove the allocation # as far as it's passed to a callee anyway, but still we may want some field analysis # for e.g. stack allocation or some other IPO optimizations - TOP_ALIAS_ESCAPES) + #=AliasEscapes=#TOP_ALIAS_ESCAPES) if !arginfo.ReturnEscape # if this is simply passed as the call argument, we can discard the `ReturnEscape` @@ -871,7 +886,7 @@ function from_interprocedural(arginfo::EscapeLatticeCache, retinfo::EscapeLattic return newarginfo ⊔ retinfo end -@noinline function invalid_escape_assignment!(ir::IRCode, pc::Int) +@noinline function unexpected_assignment!(ir::IRCode, pc::Int) @eval Main (ir = $ir; pc = $pc) error("unexpected assignment found: inspect `Main.pc` and `Main.pc`") end @@ -905,7 +920,7 @@ function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) @goto conservative_propagation end - if !(astate.ir.stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0) + if !(getinst(astate.ir, pc)[:flag] & IR_FLAG_EFFECT_FREE ≠ 0) add_thrown_escapes!(astate, pc, args) end end @@ -914,7 +929,6 @@ function escape_field!(astate::AnalysisState, @nospecialize(v), xf::FieldEscape) estate = astate.estate for xidx in xf x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape - add_escape_change!(astate, v, estate[x]) add_alias_change!(astate, v, x) end end @@ -962,11 +976,12 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) # end end # NOTE array allocations might have been proven as nothrow (https://github.com/JuliaLang/julia/pull/43565) - if !(astate.ir.stmts[pc][:flag] & IR_FLAG_EFFECT_FREE ≠ 0) - add_escape_change!(astate, name, ThrownEscape(pc)) - for i in 6:5+foreigncall_nargs - add_escape_change!(astate, args[i], ThrownEscape(pc)) - end + info = astate.ir.stmts[pc][:flag] & IR_FLAG_EFFECT_FREE ≠ 0 ? + EscapeLattice(NoEscape(), #=AliasEscapes=#true) : + EscapeLattice(ThrownEscape(pc), #=AliasEscapes=#true) + add_escape_change!(astate, name, info) + for i in 6:5+foreigncall_nargs + add_escape_change!(astate, args[i], info) end end @@ -1005,7 +1020,7 @@ function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) # we escape statements with the `ThrownEscape` property using the effect-freeness # computed by `stmt_effect_free` invoked within inlining # TODO throwness ≠ "effect-free-ness" - if !(astate.ir.stmts.flag[pc] & IR_FLAG_EFFECT_FREE ≠ 0) + if !(getinst(astate.ir, pc)[:flag] & IR_FLAG_EFFECT_FREE ≠ 0) add_thrown_escapes!(astate, pc, args, 2) end end @@ -1025,20 +1040,15 @@ function escape_builtin!(::typeof(ifelse), astate::AnalysisState, pc::Int, args: length(args) == 4 || return false f, cond, th, el = args ret = SSAValue(pc) - info = astate.estate[ret] condt = argextype(cond, astate.ir) if isa(condt, Const) && (cond = condt.val; isa(cond, Bool)) if cond - add_escape_change!(astate, th, info) add_alias_change!(astate, th, ret) else - add_escape_change!(astate, el, info) add_alias_change!(astate, el, ret) end else - add_escape_change!(astate, th, info) add_alias_change!(astate, th, ret) - add_escape_change!(astate, el, info) add_alias_change!(astate, el, ret) end return false @@ -1048,8 +1058,6 @@ function escape_builtin!(::typeof(typeassert), astate::AnalysisState, pc::Int, a length(args) == 3 || return false f, obj, typ = args ret = SSAValue(pc) - info = astate.estate[ret] - add_escape_change!(astate, obj, info) add_alias_change!(astate, ret, obj) return false end @@ -1260,7 +1268,6 @@ function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, arg AliasEscapes = copy(AliasEscapes) @label record_element_escape push!(AliasEscapes, iridx(ret, estate)) - inbounds || push!(AliasEscapes, SSAValue(pc)) # record possible BoundsError at this arrayref add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) else # this object has been used as struct, but it is used as array here (thus should throw) @@ -1310,15 +1317,7 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg @label conservative_propagation add_escape_change!(astate, val, aryinfo) elseif isa(AliasEscapes, ArrayEscapes) - for xidx in AliasEscapes - if isa(xidx, Int) - x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape - add_escape_change!(astate, val, estate[x]) - add_alias_change!(astate, val, x) - else - add_escape_change!(astate, val, ThrownEscape((xidx::SSAValue).id)) - end - end + escape_elements!(astate, val, AliasEscapes) @label add_ary_escape add_escape_change!(astate, val, ignore_aliasescapes(aryinfo)) else @@ -1334,6 +1333,14 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg return true end +function escape_elements!(astate::AnalysisState, @nospecialize(v), xa::ArrayEscapes) + estate = astate.estate + for xidx in xa + x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape + add_alias_change!(astate, v, x) + end +end + function escape_builtin!(::typeof(arraysize), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) == 3 || return false ary = args[2] @@ -1439,12 +1446,12 @@ end # end escape_builtin!(::typeof(arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = - is_safe_immutable_array_op(Array, astate, pc, args) + is_safe_immutable_array_op(Array, astate, args) escape_builtin!(::typeof(mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = - is_safe_immutable_array_op(Array, astate, pc, args) + is_safe_immutable_array_op(Array, astate, args) escape_builtin!(::typeof(arraythaw), astate::AnalysisState, pc::Int, args::Vector{Any}) = - is_safe_immutable_array_op(ImmutableArray, astate, pc, args) -function is_safe_immutable_array_op(@nospecialize(arytype), astate::AnalysisState, pc::Int, args::Vector{Any}) + is_safe_immutable_array_op(ImmutableArray, astate, args) +function is_safe_immutable_array_op(@nospecialize(arytype), astate::AnalysisState, args::Vector{Any}) length(args) == 2 || return false argextype(args[2], astate.ir) ⊑ₜ arytype || return false return true @@ -1453,6 +1460,8 @@ end # NOTE define fancy package utilities when developing EA as an external package if _TOP_MOD !== Core.Compiler include(@__MODULE__, "EAUtils.jl") + using .EAUtils: code_escapes, @code_escapes + export code_escapes, @code_escapes end end # baremodule EscapeAnalysis From 7b3c099f609607f1b75476dd2987917727705fc6 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 15 Jan 2022 12:43:50 +0900 Subject: [PATCH 35/41] export `ImmutableArray`, define `ImmutableVector` alias --- base/array.jl | 9 ++++++++- base/exports.jl | 2 ++ test/compiler/immutablearray.jl | 3 +-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index 0b7b5999e3012..696a54e795aba 100644 --- a/base/array.jl +++ b/base/array.jl @@ -119,13 +119,20 @@ Union type of [`DenseVector{T}`](@ref) and [`DenseMatrix{T}`](@ref). const DenseVecOrMat{T} = Union{DenseVector{T}, DenseMatrix{T}} """ - ImmutableArray + ImmutableArray{T,N} <: AbstractArray{T,N} Dynamically allocated, immutable array. """ const ImmutableArray = Core.ImmutableArray +""" + ImmutableVector{T} <: AbstractVector{T} + +Dynamically allocated, immutable vector. +""" +const ImmutableVector{T} = ImmutableArray{T,1} + """ IMArray{T,N} diff --git a/base/exports.jl b/base/exports.jl index 4daddd664a21a..1a3fc4a7356a6 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -22,6 +22,7 @@ export AbstractVector, AbstractVecOrMat, Array, + ImmutableArray, AbstractMatch, AbstractPattern, AbstractDict, @@ -95,6 +96,7 @@ export Val, VecOrMat, Vector, + ImmutableVector, VersionNumber, WeakKeyDict, diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index da3d327144370..520d82de80844 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -1,8 +1,7 @@ using Test -import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw +import Core: arrayfreeze, mutating_arrayfreeze, arraythaw import Core.Compiler: arrayfreeze_tfunc, mutating_arrayfreeze_tfunc, arraythaw_tfunc -const ImmutableVector{T} = ImmutableArray{T,1} @testset "ImmutableArray tfuncs" begin @test arrayfreeze_tfunc(Vector{Int}) === ImmutableVector{Int} @test arrayfreeze_tfunc(Vector) === ImmutableVector From d494cf21937f4968af1e67f4c2f89e4b1c11a66e Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 15 Jan 2022 12:44:09 +0900 Subject: [PATCH 36/41] use `IndexLinear` for immutable array --- base/indices.jl | 1 + test/immutablearray.jl | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base/indices.jl b/base/indices.jl index 28028f23c72a3..0b7d7d7212940 100644 --- a/base/indices.jl +++ b/base/indices.jl @@ -95,6 +95,7 @@ IndexStyle(A::AbstractArray) = IndexStyle(typeof(A)) IndexStyle(::Type{Union{}}) = IndexLinear() IndexStyle(::Type{<:AbstractArray}) = IndexCartesian() IndexStyle(::Type{<:Array}) = IndexLinear() +IndexStyle(::Type{<:Core.ImmutableArray}) = IndexLinear() IndexStyle(::Type{<:AbstractRange}) = IndexLinear() IndexStyle(A::AbstractArray, B::AbstractArray) = IndexStyle(IndexStyle(A), IndexStyle(B)) diff --git a/test/immutablearray.jl b/test/immutablearray.jl index e0f3c67ec5312..c7fce37cd8a6f 100644 --- a/test/immutablearray.jl +++ b/test/immutablearray.jl @@ -1,5 +1,5 @@ using Test -import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw +import Core: arrayfreeze, mutating_arrayfreeze, arraythaw @testset "basic ImmutableArray functionality" begin eltypes = (Float16, Float32, Float64, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128) @@ -23,8 +23,8 @@ import Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw @test axes(a) == axes(b) @test strides(a) == strides(b) @test keys(a) == keys(b) - @test_broken IndexStyle(a) == IndexStyle(b) # ImmutableArray is IndexCartesian whereas Array is IndexLinear - worth looking into - @test_broken eachindex(a) == eachindex(b) + @test IndexStyle(a) == IndexStyle(b) # ImmutableArray is IndexCartesian whereas Array is IndexLinear - worth looking into + @test eachindex(a) == eachindex(b) end end @@ -52,4 +52,4 @@ end @test_throws ArgumentError arraythaw([1,2,3], nothing) @test_throws TypeError arraythaw(a) @test_throws TypeError arraythaw("not an array") -end \ No newline at end of file +end From de410f10450a8b9d3737e2ca539b262c9380f2d0 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Sat, 15 Jan 2022 12:44:21 +0900 Subject: [PATCH 37/41] enable `test/immutablearray` testset --- test/choosetests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/choosetests.jl b/test/choosetests.jl index 7e02de28525b0..66d87232d02d0 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -23,7 +23,7 @@ const TESTNAMES = [ "errorshow", "sets", "goto", "llvmcall", "llvmcall2", "ryu", "some", "meta", "stacktraces", "docs", "misc", "threads", "stress", "binaryplatforms", "atexit", - "enums", "cmdlineargs", "int", "interpreter", + "enums", "cmdlineargs", "immutablearray", "int", "interpreter", "checked", "bitset", "floatfuncs", "precompile", "boundscheck", "error", "ambiguous", "cartesian", "osutils", "channels", "iostream", "secretbuffer", "specificity", From 283c36b4d56c4b4b93a574ec43cea8afe6a5426a Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 18 Jan 2022 15:30:56 +0900 Subject: [PATCH 38/41] fix tests --- stdlib/LinearAlgebra/test/adjtrans.jl | 16 ++++++------- stdlib/LinearAlgebra/test/bidiag.jl | 16 ++++++------- stdlib/LinearAlgebra/test/diagonal.jl | 10 ++++---- stdlib/LinearAlgebra/test/hessenberg.jl | 10 ++++---- stdlib/LinearAlgebra/test/symmetric.jl | 14 ++++++------ stdlib/LinearAlgebra/test/triangular.jl | 14 ++++++------ stdlib/LinearAlgebra/test/tridiag.jl | 18 +++++++-------- test/testhelpers/ImmutableArrays.jl | 28 ----------------------- test/testhelpers/SimpleImmutableArrays.jl | 28 +++++++++++++++++++++++ 9 files changed, 77 insertions(+), 77 deletions(-) delete mode 100644 test/testhelpers/ImmutableArrays.jl create mode 100644 test/testhelpers/SimpleImmutableArrays.jl diff --git a/stdlib/LinearAlgebra/test/adjtrans.jl b/stdlib/LinearAlgebra/test/adjtrans.jl index 7b782d463768d..ae2946a68809a 100644 --- a/stdlib/LinearAlgebra/test/adjtrans.jl +++ b/stdlib/LinearAlgebra/test/adjtrans.jl @@ -241,22 +241,22 @@ end @test convert(Transpose{Float64,Matrix{Float64}}, Transpose(intmat))::Transpose{Float64,Matrix{Float64}} == Transpose(intmat) end -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Adjoint and Transpose convert methods to AbstractArray" begin # tests corresponding to #34995 intvec, intmat = [1, 2], [1 2 3; 4 5 6] - statvec = ImmutableArray(intvec) - statmat = ImmutableArray(intmat) + statvec = SimpleImmutableArray(intvec) + statmat = SimpleImmutableArray(intmat) - @test convert(AbstractArray{Float64}, Adjoint(statvec))::Adjoint{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Adjoint(statvec) + @test convert(AbstractArray{Float64}, Adjoint(statvec))::Adjoint{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Adjoint(statvec) @test convert(AbstractArray{Float64}, Adjoint(statmat))::Array{Float64,2} == Adjoint(statmat) - @test convert(AbstractArray{Float64}, Transpose(statvec))::Transpose{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Transpose(statvec) + @test convert(AbstractArray{Float64}, Transpose(statvec))::Transpose{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Transpose(statvec) @test convert(AbstractArray{Float64}, Transpose(statmat))::Array{Float64,2} == Transpose(statmat) - @test convert(AbstractMatrix{Float64}, Adjoint(statvec))::Adjoint{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Adjoint(statvec) + @test convert(AbstractMatrix{Float64}, Adjoint(statvec))::Adjoint{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Adjoint(statvec) @test convert(AbstractMatrix{Float64}, Adjoint(statmat))::Array{Float64,2} == Adjoint(statmat) - @test convert(AbstractMatrix{Float64}, Transpose(statvec))::Transpose{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Transpose(statvec) + @test convert(AbstractMatrix{Float64}, Transpose(statvec))::Transpose{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Transpose(statvec) @test convert(AbstractMatrix{Float64}, Transpose(statmat))::Array{Float64,2} == Transpose(statmat) end diff --git a/stdlib/LinearAlgebra/test/bidiag.jl b/stdlib/LinearAlgebra/test/bidiag.jl index 58de45e9e525c..182b1dbe02ede 100644 --- a/stdlib/LinearAlgebra/test/bidiag.jl +++ b/stdlib/LinearAlgebra/test/bidiag.jl @@ -668,20 +668,20 @@ end @test c \ A ≈ c \ Matrix(A) end -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Conversion to AbstractArray" begin # tests corresponding to #34995 - dv = ImmutableArray([1, 2, 3, 4]) - ev = ImmutableArray([7, 8, 9]) + dv = SimpleImmutableArray([1, 2, 3, 4]) + ev = SimpleImmutableArray([7, 8, 9]) Bu = Bidiagonal(dv, ev, :U) Bl = Bidiagonal(dv, ev, :L) - @test convert(AbstractArray{Float64}, Bu)::Bidiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Bu - @test convert(AbstractMatrix{Float64}, Bu)::Bidiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Bu - @test convert(AbstractArray{Float64}, Bl)::Bidiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Bl - @test convert(AbstractMatrix{Float64}, Bl)::Bidiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Bl + @test convert(AbstractArray{Float64}, Bu)::Bidiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Bu + @test convert(AbstractMatrix{Float64}, Bu)::Bidiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Bu + @test convert(AbstractArray{Float64}, Bl)::Bidiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Bl + @test convert(AbstractMatrix{Float64}, Bl)::Bidiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Bl end @testset "block-bidiagonal matrix indexing" begin diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 6f4aae5358a39..f2c5c619ab66d 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -910,16 +910,16 @@ end end const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Conversion to AbstractArray" begin # tests corresponding to #34995 - d = ImmutableArray([1, 2, 3, 4]) + d = SimpleImmutableArray([1, 2, 3, 4]) D = Diagonal(d) - @test convert(AbstractArray{Float64}, D)::Diagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == D - @test convert(AbstractMatrix{Float64}, D)::Diagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == D + @test convert(AbstractArray{Float64}, D)::Diagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == D + @test convert(AbstractMatrix{Float64}, D)::Diagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == D end @testset "divisions functionality" for elty in (Int, Float64, ComplexF64) diff --git a/stdlib/LinearAlgebra/test/hessenberg.jl b/stdlib/LinearAlgebra/test/hessenberg.jl index 9b623273666c2..1c93359bad6bb 100644 --- a/stdlib/LinearAlgebra/test/hessenberg.jl +++ b/stdlib/LinearAlgebra/test/hessenberg.jl @@ -194,16 +194,16 @@ end end end -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Conversion to AbstractArray" begin # tests corresponding to #34995 - A = ImmutableArray([1 2 3; 4 5 6; 7 8 9]) + A = SimpleImmutableArray([1 2 3; 4 5 6; 7 8 9]) H = UpperHessenberg(A) - @test convert(AbstractArray{Float64}, H)::UpperHessenberg{Float64,ImmutableArray{Float64,2,Array{Float64,2}}} == H - @test convert(AbstractMatrix{Float64}, H)::UpperHessenberg{Float64,ImmutableArray{Float64,2,Array{Float64,2}}} == H + @test convert(AbstractArray{Float64}, H)::UpperHessenberg{Float64,SimpleImmutableArray{Float64,2,Array{Float64,2}}} == H + @test convert(AbstractMatrix{Float64}, H)::UpperHessenberg{Float64,SimpleImmutableArray{Float64,2,Array{Float64,2}}} == H end end # module TestHessenberg diff --git a/stdlib/LinearAlgebra/test/symmetric.jl b/stdlib/LinearAlgebra/test/symmetric.jl index 47a36df5e7883..55af4fe456ff2 100644 --- a/stdlib/LinearAlgebra/test/symmetric.jl +++ b/stdlib/LinearAlgebra/test/symmetric.jl @@ -544,19 +544,19 @@ end end const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Conversion to AbstractArray" begin # tests corresponding to #34995 - immutablemat = ImmutableArray([1 2 3; 4 5 6; 7 8 9]) + immutablemat = SimpleImmutableArray([1 2 3; 4 5 6; 7 8 9]) for SymType in (Symmetric, Hermitian) S = Float64 symmat = SymType(immutablemat) - @test convert(AbstractArray{S}, symmat).data isa ImmutableArray{S} - @test convert(AbstractMatrix{S}, symmat).data isa ImmutableArray{S} - @test AbstractArray{S}(symmat).data isa ImmutableArray{S} - @test AbstractMatrix{S}(symmat).data isa ImmutableArray{S} + @test convert(AbstractArray{S}, symmat).data isa SimpleImmutableArray{S} + @test convert(AbstractMatrix{S}, symmat).data isa SimpleImmutableArray{S} + @test AbstractArray{S}(symmat).data isa SimpleImmutableArray{S} + @test AbstractMatrix{S}(symmat).data isa SimpleImmutableArray{S} @test convert(AbstractArray{S}, symmat) == symmat @test convert(AbstractMatrix{S}, symmat) == symmat end diff --git a/stdlib/LinearAlgebra/test/triangular.jl b/stdlib/LinearAlgebra/test/triangular.jl index b53100c6fc654..72959910dab1a 100644 --- a/stdlib/LinearAlgebra/test/triangular.jl +++ b/stdlib/LinearAlgebra/test/triangular.jl @@ -689,20 +689,20 @@ let A = UpperTriangular([Furlong(1) Furlong(4); Furlong(0) Furlong(1)]) @test sqrt(A) == Furlong{1//2}.(UpperTriangular([1 2; 0 1])) end -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "AbstractArray constructor should preserve underlying storage type" begin # tests corresponding to #34995 local m = 4 local T, S = Float32, Float64 - immutablemat = ImmutableArray(randn(T,m,m)) + immutablemat = SimpleImmutableArray(randn(T,m,m)) for TriType in (UpperTriangular, LowerTriangular, UnitUpperTriangular, UnitLowerTriangular) trimat = TriType(immutablemat) - @test convert(AbstractArray{S}, trimat).data isa ImmutableArray{S} - @test convert(AbstractMatrix{S}, trimat).data isa ImmutableArray{S} - @test AbstractArray{S}(trimat).data isa ImmutableArray{S} - @test AbstractMatrix{S}(trimat).data isa ImmutableArray{S} + @test convert(AbstractArray{S}, trimat).data isa SimpleImmutableArray{S} + @test convert(AbstractMatrix{S}, trimat).data isa SimpleImmutableArray{S} + @test AbstractArray{S}(trimat).data isa SimpleImmutableArray{S} + @test AbstractMatrix{S}(trimat).data isa SimpleImmutableArray{S} @test convert(AbstractArray{S}, trimat) == trimat @test convert(AbstractMatrix{S}, trimat) == trimat end diff --git a/stdlib/LinearAlgebra/test/tridiag.jl b/stdlib/LinearAlgebra/test/tridiag.jl index ecdf6b416baa5..fa6ce93fc296c 100644 --- a/stdlib/LinearAlgebra/test/tridiag.jl +++ b/stdlib/LinearAlgebra/test/tridiag.jl @@ -667,21 +667,21 @@ end @test ishermitian(S) end -isdefined(Main, :ImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "ImmutableArrays.jl")) -using .Main.ImmutableArrays +isdefined(Main, :SimpleImmutableArrays) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "SimpleImmutableArrays.jl")) +using .Main.SimpleImmutableArrays @testset "Conversion to AbstractArray" begin # tests corresponding to #34995 - v1 = ImmutableArray([1, 2]) - v2 = ImmutableArray([3, 4, 5]) - v3 = ImmutableArray([6, 7]) + v1 = SimpleImmutableArray([1, 2]) + v2 = SimpleImmutableArray([3, 4, 5]) + v3 = SimpleImmutableArray([6, 7]) T = Tridiagonal(v1, v2, v3) Tsym = SymTridiagonal(v2, v1) - @test convert(AbstractArray{Float64}, T)::Tridiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == T - @test convert(AbstractMatrix{Float64}, T)::Tridiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == T - @test convert(AbstractArray{Float64}, Tsym)::SymTridiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Tsym - @test convert(AbstractMatrix{Float64}, Tsym)::SymTridiagonal{Float64,ImmutableArray{Float64,1,Array{Float64,1}}} == Tsym + @test convert(AbstractArray{Float64}, T)::Tridiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == T + @test convert(AbstractMatrix{Float64}, T)::Tridiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == T + @test convert(AbstractArray{Float64}, Tsym)::SymTridiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Tsym + @test convert(AbstractMatrix{Float64}, Tsym)::SymTridiagonal{Float64,SimpleImmutableArray{Float64,1,Array{Float64,1}}} == Tsym end @testset "dot(x,A,y) for A::Tridiagonal or SymTridiagonal" begin diff --git a/test/testhelpers/ImmutableArrays.jl b/test/testhelpers/ImmutableArrays.jl deleted file mode 100644 index df2a78387e07b..0000000000000 --- a/test/testhelpers/ImmutableArrays.jl +++ /dev/null @@ -1,28 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# ImmutableArrays (arrays that implement getindex but not setindex!) - -# This test file defines an array wrapper that is immutable. It can be used to -# test the action of methods on immutable arrays. - -module ImmutableArrays - -export ImmutableArray - -"An immutable wrapper type for arrays." -struct ImmutableArray{T,N,A<:AbstractArray} <: AbstractArray{T,N} - data::A -end - -ImmutableArray(data::AbstractArray{T,N}) where {T,N} = ImmutableArray{T,N,typeof(data)}(data) - -# Minimal AbstractArray interface -Base.size(A::ImmutableArray) = size(A.data) -Base.size(A::ImmutableArray, d) = size(A.data, d) -Base.getindex(A::ImmutableArray, i...) = getindex(A.data, i...) - -# The immutable array remains immutable after conversion to AbstractArray -AbstractArray{T}(A::ImmutableArray) where {T} = ImmutableArray(AbstractArray{T}(A.data)) -AbstractArray{T,N}(A::ImmutableArray{S,N}) where {S,T,N} = ImmutableArray(AbstractArray{T,N}(A.data)) - -end diff --git a/test/testhelpers/SimpleImmutableArrays.jl b/test/testhelpers/SimpleImmutableArrays.jl new file mode 100644 index 0000000000000..7276020310b89 --- /dev/null +++ b/test/testhelpers/SimpleImmutableArrays.jl @@ -0,0 +1,28 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# SimpleImmutableArrays (arrays that implement getindex but not setindex!) + +# This test file defines an array wrapper that is immutable. It can be used to +# test the action of methods on immutable arrays. + +module SimpleImmutableArrays + +export SimpleImmutableArray + +"An immutable wrapper type for arrays." +struct SimpleImmutableArray{T,N,A<:AbstractArray} <: AbstractArray{T,N} + data::A +end + +SimpleImmutableArray(data::AbstractArray{T,N}) where {T,N} = SimpleImmutableArray{T,N,typeof(data)}(data) + +# Minimal AbstractArray interface +Base.size(A::SimpleImmutableArray) = size(A.data) +Base.size(A::SimpleImmutableArray, d) = size(A.data, d) +Base.getindex(A::SimpleImmutableArray, i...) = getindex(A.data, i...) + +# The immutable array remains immutable after conversion to AbstractArray +AbstractArray{T}(A::SimpleImmutableArray) where {T} = SimpleImmutableArray(AbstractArray{T}(A.data)) +AbstractArray{T,N}(A::SimpleImmutableArray{S,N}) where {S,T,N} = SimpleImmutableArray(AbstractArray{T,N}(A.data)) + +end From 2560b732adcfae22eebdd56071b6b544e4944640 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Thu, 20 Jan 2022 13:20:23 +0900 Subject: [PATCH 39/41] update EA --- base/compiler/EscapeAnalysis/EAUtils.jl | 10 +- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 620 +++++++++--------- 2 files changed, 330 insertions(+), 300 deletions(-) diff --git a/base/compiler/EscapeAnalysis/EAUtils.jl b/base/compiler/EscapeAnalysis/EAUtils.jl index 78ebf5eff206f..cde0c8257a7d3 100644 --- a/base/compiler/EscapeAnalysis/EAUtils.jl +++ b/base/compiler/EscapeAnalysis/EAUtils.jl @@ -25,7 +25,7 @@ import InteractiveUtils: gen_call_with_extracted_types_and_kwargs Evaluates the arguments to the function call, determines its types, and then calls [`code_escapes`](@ref) on the resulting expression. As with `@code_typed` and its family, any of `code_escapes` keyword arguments can be given -as the optional arguments like `@code_escpase interp=myinterp myfunc(myargs...)`. +as the optional arguments like `@code_escapes interp=myinterp myfunc(myargs...)`. """ macro code_escapes(ex0...) return gen_call_with_extracted_types_and_kwargs(__module__, :code_escapes, ex0) @@ -36,7 +36,7 @@ end # @static if EA_AS_PKG code_escapes(f, argtypes=Tuple{}; [world], [interp]) -> result::EscapeResult code_escapes(tt::Type{<:Tuple}; [world], [interp]) -> result::EscapeResult -Runs the escape analysis on optimized IR of a genefic function call with the given type signature. +Runs the escape analysis on optimized IR of a generic function call with the given type signature. Note that the escape analysis runs after inlining, but before any other optimizations. ```julia @@ -252,9 +252,7 @@ function run_passes_with_ea(interp::EscapeAnalyzer, ci::CodeInfo, sv::Optimizati @timeit "Inlining" ir = ssa_inlining_pass!(ir, ir.linetable, sv.inlining, ci.propagate_inbounds) # @timeit "verify 2" verify_ir(ir) @timeit "compact 2" ir = compact!(ir) - nargs = let def = sv.linfo.def - isa(def, Method) ? Int(def.nargs) : 0 - end + nargs = let def = sv.linfo.def; isa(def, Method) ? Int(def.nargs) : 0; end local state try @timeit "collect escape information" state = analyze_escapes(ir, nargs) @@ -298,7 +296,7 @@ function get_name_color(x::EscapeLattice, symbol::Bool = false) name, color = (getname(EA.NoEscape), "✓"), :green elseif EA.has_all_escape(x) name, color = (getname(EA.AllEscape), "X"), :red - elseif EA.NoEscape() ⊏ (EA.ignore_thrownescapes ∘ EA.ignore_aliasescapes)(x) ⊑ EA.AllReturnEscape() + elseif EA.NoEscape() ⊏ (EA.ignore_thrownescapes ∘ EA.ignore_aliasinfo)(x) ⊑ EA.AllReturnEscape() name = (getname(EA.ReturnEscape), "↑") color = EA.has_thrown_escape(x) ? :yellow : :cyan else diff --git a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl index 1e63c2e07e366..d314c33c9004b 100644 --- a/base/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -6,10 +6,7 @@ export has_no_escape, has_return_escape, has_thrown_escape, - has_all_escape, - is_load_forwardable, - is_sroa_eligible, - can_elide_finalizer + has_all_escape # analysis # ======== @@ -40,10 +37,22 @@ else include(@__MODULE__, "compiler/EscapeAnalysis/disjoint_set.jl") end -# XXX better to be IdSet{Int}? -const FieldEscape = BitSet -const FieldEscapes = Vector{BitSet} -const ArrayEscapes = IdSet{Int} +const AInfo = BitSet # XXX better to be IdSet{Int}? +struct Indexable + array::Bool + infos::Vector{AInfo} +end +struct Unindexable + array::Bool + info::AInfo +end +function merge_to_unindexable(info::AInfo, infos::Vector{AInfo}) + for i = 1:length(infos) + info = info ∪ infos[i] + end + return info +end +merge_to_unindexable(infos::Vector{AInfo}) = merge_to_unindexable(AInfo(), infos) """ x::EscapeLattice @@ -55,14 +64,14 @@ A lattice for escape information, which holds the following properties: simply because it's passed as call argument - `x.ThrownEscape::BitSet`: records SSA statements where `x` can be thrown as exception: this information will be used by `escape_exception!` to propagate potential escapes via exception -- `x.AliasEscapes::Union{FieldEscapes,ArrayEscapes,Bool}`: maintains all possible values +- `x.AliasInfo::Union{Indexable,Unindexable,Bool}`: maintains all possible values that can be aliased to fields or array elements of `x`: - * `x.AliasEscapes === false` indicates the fields/elements of `x` isn't analyzed yet - * `x.AliasEscapes === true` indicates the fields/elements of `x` can't be analyzed, + * `x.AliasInfo === false` indicates the fields/elements of `x` isn't analyzed yet + * `x.AliasInfo === true` indicates the fields/elements of `x` can't be analyzed, e.g. the type of `x` is not known or is not concrete and thus its fields/elements can't be known precisely - * `x.AliasEscapes::FieldEscapes` records all the possible values that can be aliased fields of object `x`, - * `x.AliasEscapes::ArrayEscapes` records all the possible values that be aliased to elements of array `x` + * `x.AliasInfo::Indexable` records all the possible values that can be aliased to fields/elements of `x` with precise index information + * `x.AliasInfo::Unindexable` records all the possible values that can be aliased to fields/elements of `x` without precise index information - `x.ArgEscape::Int` (not implemented yet): indicates it will escape to the caller through `setfield!` on argument(s) * `-1` : no escape @@ -85,67 +94,64 @@ struct EscapeLattice Analyzed::Bool ReturnEscape::BitSet ThrownEscape::BitSet - AliasEscapes #::Union{FieldEscapes,ArrayEscapes,Bool} + AliasInfo #::Union{Indexable,Unindexable,Bool} # TODO: ArgEscape::Int function EscapeLattice( Analyzed::Bool, ReturnEscape::BitSet, ThrownEscape::BitSet, - AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=#, + AliasInfo#=::Union{Indexable,Unindexable,Bool}=#, ) - @nospecialize AliasEscapes + @nospecialize AliasInfo return new( Analyzed, ReturnEscape, ThrownEscape, - AliasEscapes, + AliasInfo, ) end function EscapeLattice( x::EscapeLattice, # non-concrete fields should be passed as default arguments # in order to avoid allocating non-concrete `NamedTuple`s - AliasEscapes#=::Union{FieldEscapes,ArrayEscapes,Bool}=# = x.AliasEscapes; + AliasInfo#=::Union{Indexable,Unindexable,Bool}=# = x.AliasInfo; Analyzed::Bool = x.Analyzed, ReturnEscape::BitSet = x.ReturnEscape, ThrownEscape::BitSet = x.ThrownEscape, ) - @nospecialize AliasEscapes + @nospecialize AliasInfo return new( Analyzed, ReturnEscape, ThrownEscape, - AliasEscapes, + AliasInfo, ) end end # precomputed default values in order to eliminate computations at each callsite -const BOT_RETURN_ESCAPE = BitSet() +const BOT_RETURN_ESCAPE = const BOT_THROWN_ESCAPE = BitSet() +const TOP_RETURN_ESCAPE = const TOP_THROWN_ESCAPE = BitSet(0:100_000) const ARG_RETURN_ESCAPE = BitSet(0) -const TOP_RETURN_ESCAPE = BitSet(0:100_000) -const BOT_THROWN_ESCAPE = BitSet() -const TOP_THROWN_ESCAPE = BitSet(0:100_000) - -const BOT_ALIAS_ESCAPES = false -const TOP_ALIAS_ESCAPES = true +const BOT_ALIAS_INFO = false +const TOP_ALIAS_INFO = true # the constructors -NotAnalyzed() = EscapeLattice(false, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) # not formally part of the lattice -NoEscape() = EscapeLattice(true, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) -ReturnEscape(pc::Int) = EscapeLattice(true, BitSet(pc), BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) -ArgumentReturnEscape() = EscapeLattice(true, ARG_RETURN_ESCAPE, BOT_THROWN_ESCAPE, TOP_ALIAS_ESCAPES) # TODO allow interprocedural field analysis? -AllReturnEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_ESCAPES) -ThrownEscape(pc::Int) = EscapeLattice(true, BOT_RETURN_ESCAPE, BitSet(pc), BOT_ALIAS_ESCAPES) -ThrownEscape(pcs::BitSet) = EscapeLattice(true, BOT_RETURN_ESCAPE, pcs, BOT_ALIAS_ESCAPES) -AllEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, TOP_THROWN_ESCAPE, TOP_ALIAS_ESCAPES) +NotAnalyzed() = EscapeLattice(false, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_INFO) # not formally part of the lattice +NoEscape() = EscapeLattice(true, BOT_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_INFO) +ReturnEscape(pc::Int) = EscapeLattice(true, BitSet(pc), BOT_THROWN_ESCAPE, BOT_ALIAS_INFO) +ArgumentReturnEscape() = EscapeLattice(true, ARG_RETURN_ESCAPE, BOT_THROWN_ESCAPE, TOP_ALIAS_INFO) # TODO allow interprocedural field analysis? +AllReturnEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, BOT_THROWN_ESCAPE, BOT_ALIAS_INFO) +ThrownEscape(pc::Int) = EscapeLattice(true, BOT_RETURN_ESCAPE, BitSet(pc), BOT_ALIAS_INFO) +ThrownEscape(pcs::BitSet) = EscapeLattice(true, BOT_RETURN_ESCAPE, pcs, BOT_ALIAS_INFO) +AllEscape() = EscapeLattice(true, TOP_RETURN_ESCAPE, TOP_THROWN_ESCAPE, TOP_ALIAS_INFO) const ⊥, ⊤ = NotAnalyzed(), AllEscape() # Convenience names for some ⊑ queries -has_no_escape(x::EscapeLattice) = ignore_aliasescapes(x) ⊑ NoEscape() +has_no_escape(x::EscapeLattice) = ignore_aliasinfo(x) ⊑ NoEscape() has_return_escape(x::EscapeLattice) = !isempty(x.ReturnEscape) has_return_escape(x::EscapeLattice, pc::Int) = pc in x.ReturnEscape has_thrown_escape(x::EscapeLattice) = !isempty(x.ThrownEscape) @@ -154,46 +160,14 @@ has_all_escape(x::EscapeLattice) = ⊤ ⊑ x # utility lattice constructors ignore_thrownescapes(x::EscapeLattice) = EscapeLattice(x; ThrownEscape=BOT_THROWN_ESCAPE) -ignore_aliasescapes(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_ESCAPES) - -""" - is_load_forwardable(x::EscapeLattice) -> Bool - -Queries if `x` is elibigle for store-to-load forwarding optimization. -""" -function is_load_forwardable(x::EscapeLattice) - if x.AliasEscapes === false || # allows this query to work for immutables since we don't impose escape on them - isa(x.AliasEscapes, FieldEscapes) - # NOTE technically we also need to check `!has_thrown_escape(x)` here as well, - # but we can also do equivalent check during forwarding - return true - end - return false -end - -""" - is_sroa_eligible(x::EscapeLattice) -> Bool - -Queries allocation eliminability by SROA. -""" -is_sroa_eligible(x::EscapeLattice) = is_load_forwardable(x) && !has_return_escape(x) - -""" - can_elide_finalizer(x::EscapeLattice, pc::Int) -> Bool - -Queries the validity of the finalizer elision optimization at the return site of SSA statement `pc`, -which inserts `finalize` call when the lifetime of interested object ends. -Note that we don't need to take `x.ThrownEscape` into account because it would have never -been thrown when the program execution reaches the `return` site. -""" -can_elide_finalizer(x::EscapeLattice, pc::Int) = - !(has_return_escape(x, 0) || has_return_escape(x, pc)) +ignore_aliasinfo(x::EscapeLattice) = EscapeLattice(x, BOT_ALIAS_INFO) # we need to make sure this `==` operator corresponds to lattice equality rather than object equality, # otherwise `propagate_changes` can't detect the convergence x::EscapeLattice == y::EscapeLattice = begin # fast pass: better to avoid top comparison x === y && return true + x.Analyzed === y.Analyzed || return false xr, yr = x.ReturnEscape, y.ReturnEscape if xr === TOP_RETURN_ESCAPE yr === TOP_RETURN_ESCAPE || return false @@ -210,18 +184,20 @@ x::EscapeLattice == y::EscapeLattice = begin else xt == yt || return false end - xf, yf = x.AliasEscapes, y.AliasEscapes - if isa(xf, Bool) - xf === yf || return false - elseif isa(xf, FieldEscapes) - isa(yf, FieldEscapes) || return false - xf == yf || return false + xa, ya = x.AliasInfo, y.AliasInfo + if isa(xa, Bool) + xa === ya || return false + elseif isa(xa, Indexable) + isa(ya, Indexable) || return false + xa.array === ya.array || return false + xa.infos == ya.infos || return false else - xf = xf::ArrayEscapes - isa(yf, ArrayEscapes) || return false - xf == yf || return false + xa = xa::Unindexable + isa(ya, Unindexable) || return false + xa.array === ya.array || return false + xa.info == ya.info || return false end - return x.Analyzed === y.Analyzed + return true end """ @@ -240,6 +216,7 @@ x::EscapeLattice ⊑ y::EscapeLattice = begin elseif y === ⊥ return false # return x === ⊥ end + x.Analyzed ≤ y.Analyzed || return false xr, yr = x.ReturnEscape, y.ReturnEscape if xr === TOP_RETURN_ESCAPE yr !== TOP_RETURN_ESCAPE && return false @@ -252,28 +229,38 @@ x::EscapeLattice ⊑ y::EscapeLattice = begin elseif yt !== TOP_THROWN_ESCAPE xt ⊆ yt || return false end - xf, yf = x.AliasEscapes, y.AliasEscapes - if isa(xf, Bool) - xf && yf !== true && return false - elseif isa(xf, FieldEscapes) - if isa(yf, FieldEscapes) - xn, yn = length(xf), length(yf) + xa, ya = x.AliasInfo, y.AliasInfo + if isa(xa, Bool) + xa && ya !== true && return false + elseif isa(xa, Indexable) + if isa(ya, Indexable) + xa.array === ya.array || return false + xinfos, yinfos = xa.infos, ya.infos + xn, yn = length(xinfos), length(yinfos) xn > yn && return false for i in 1:xn - xf[i] ⊆ yf[i] || return false + xinfos[i] ⊆ yinfos[i] || return false + end + elseif isa(ya, Unindexable) + xa.array === ya.array || return false + xinfos, yinfo = xa.infos, ya.info + for i = length(xf) + xinfos[i] ⊆ yinfo || return false end else - yf === true || return false + ya === true || return false end else - xf = xf::ArrayEscapes - if isa(yf, ArrayEscapes) - xf ⊆ yf || return false + xa = xa::Unindexable + if isa(ya, Unindexable) + xa.array === ya.array || return false + xinfo, yinfo = xa.info, ya.info + xinfo ⊆ yinfo || return false else - yf === true || return false + ya === true || return false end end - return x.Analyzed ≤ y.Analyzed + return true end """ @@ -326,41 +313,53 @@ x::EscapeLattice ⊔ y::EscapeLattice = begin else ThrownEscape = xt ∪ yt end - xf, yf = x.AliasEscapes, y.AliasEscapes - if xf === true || yf === true - AliasEscapes = true - elseif xf === false - AliasEscapes = yf - elseif yf === false - AliasEscapes = xf - elseif isa(xf, FieldEscapes) - if isa(yf, FieldEscapes) - xn, yn = length(xf), length(yf) + xa, ya = x.AliasInfo, y.AliasInfo + if xa === true || ya === true + AliasInfo = true + elseif xa === false + AliasInfo = ya + elseif ya === false + AliasInfo = xa + elseif isa(xa, Indexable) + if isa(ya, Indexable) && xa.array === ya.array + xinfos, yinfos = xa.infos, ya.infos + xn, yn = length(xinfos), length(yinfos) nmax, nmin = max(xn, yn), min(xn, yn) - AliasEscapes = Vector{FieldEscape}(undef, nmax) + infos = Vector{AInfo}(undef, nmax) for i in 1:nmax if i > nmin - AliasEscapes[i] = (xn > yn ? xf : yf)[i] + infos[i] = (xn > yn ? xinfos : yinfos)[i] else - AliasEscapes[i] = xf[i] ∪ yf[i] + infos[i] = xinfos[i] ∪ yinfos[i] end end + AliasInfo = Indexable(xa.array, infos) + elseif isa(ya, Unindexable) && xa.array === ya.array + xinfos, yinfo = xa.infos, ya.info + info = merge_to_unindexable(yinfo, xinfos) + AliasInfo = Unindexable(xa.array, info) else - AliasEscapes = true # handle conflicting case conservatively + AliasInfo = true # handle conflicting case conservatively end else - xf = xf::ArrayEscapes - if isa(yf, ArrayEscapes) - AliasEscapes = xf ∪ yf + xa = xa::Unindexable + if isa(ya, Indexable) && xa.array === ya.array + xinfo, yinfos = xa.info, ya.infos + info = merge_to_unindexable(xinfo, yinfos) + AliasInfo = Unindexable(xa.array, info) + elseif isa(ya, Unindexable) && xa.array === ya.array + xinfo, yinfo = xa.info, ya.info + info = xinfo ∪ yinfo + AliasInfo = Unindexable(xa.array, info) else - AliasEscapes = true # handle conflicting case conservatively + AliasInfo = true # handle conflicting case conservatively end end return EscapeLattice( x.Analyzed | y.Analyzed, ReturnEscape, ThrownEscape, - AliasEscapes, + AliasInfo, ) end @@ -646,14 +645,6 @@ function propagate_changes!(estate::EscapeState, changes::Changes) for change in changes if isa(change, EscapeChange) anychanged |= propagate_escape_change!(estate, change) - xidx, info = change - aliases = getaliases(xidx, estate) - if aliases !== nothing - for aidx in aliases - morechange = EscapeChange(aidx, info) - anychanged |= propagate_escape_change!(estate, morechange) - end - end else anychanged |= propagate_alias_change!(estate, change) end @@ -661,10 +652,27 @@ function propagate_changes!(estate::EscapeState, changes::Changes) return anychanged end -function propagate_escape_change!(estate::EscapeState, change::EscapeChange) +@inline propagate_escape_change!(estate::EscapeState, change::EscapeChange) = + propagate_escape_change!(⊔, estate, change) + +# allows this to work as lattice join as well as lattice meet +@inline function propagate_escape_change!(@nospecialize(op), + estate::EscapeState, change::EscapeChange) xidx, info = change + anychanged = _propagate_escape_change!(op, estate, xidx, info) + aliases = getaliases(xidx, estate) + if aliases !== nothing + for aidx in aliases + anychanged |= _propagate_escape_change!(op, estate, aidx, info) + end + end + return anychanged +end + +@inline function _propagate_escape_change!(@nospecialize(op), + estate::EscapeState, xidx::Int, info::EscapeLattice) old = estate.escapes[xidx] - new = old ⊔ info + new = op(old, info) if old ≠ new estate.escapes[xidx] = new return true @@ -672,7 +680,7 @@ function propagate_escape_change!(estate::EscapeState, change::EscapeChange) return false end -function propagate_alias_change!(estate::EscapeState, change::AliasChange) +@inline function propagate_alias_change!(estate::EscapeState, change::AliasChange) xidx, yidx = change xroot = find_root!(estate.aliasset, xidx) yroot = find_root!(estate.aliasset, yidx) @@ -874,7 +882,7 @@ function from_interprocedural(arginfo::EscapeLatticeCache, retinfo::EscapeLattic # it might be okay from the SROA point of view, since we can't remove the allocation # as far as it's passed to a callee anyway, but still we may want some field analysis # for e.g. stack allocation or some other IPO optimizations - #=AliasEscapes=#TOP_ALIAS_ESCAPES) + #=AliasInfo=#TOP_ALIAS_INFO) if !arginfo.ReturnEscape # if this is simply passed as the call argument, we can discard the `ReturnEscape` @@ -894,47 +902,57 @@ end function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) obj = SSAValue(pc) objinfo = astate.estate[obj] - AliasEscapes = objinfo.AliasEscapes + AliasInfo = objinfo.AliasInfo nargs = length(args) - if isa(AliasEscapes, Bool) - @label conservative_propagation - # the fields couldn't be analyzed precisely: propagate the entire escape information - # of this object to all its fields as the most conservative propagation + if isa(AliasInfo, Bool) + @goto conservative_propagation + elseif isa(AliasInfo, Indexable) && !AliasInfo.array + # fields are known precisely: propagate escape information imposed on recorded possibilities to the exact field values + infos = AliasInfo.infos + nf = length(infos) for i in 2:nargs - add_escape_change!(astate, args[i], objinfo) + i-1 > nf && break # may happen when e.g. ϕ-node merges values with different types + escape_field!(astate, args[i], infos[i-1]) + push!(infos[i-1], -pc) # record def + # propagate the escape information of this object ignoring field information + add_escape_change!(astate, args[i], ignore_aliasinfo(objinfo)) end - elseif isa(AliasEscapes, FieldEscapes) - # fields are known: propagate escape information imposed on recorded possibilities - nf = length(AliasEscapes) + elseif isa(AliasInfo, Unindexable) && !AliasInfo.array + # fields are known partially: propagate escape information imposed on recorded possibilities to all fields values + info = AliasInfo.info for i in 2:nargs - # fields are known: propagate the escape information of this object ignoring field information - add_escape_change!(astate, args[i], ignore_aliasescapes(objinfo)) - # fields are known: propagate escape information imposed on recorded possibilities - i-1 > nf && break # may happen when e.g. ϕ-node merges values with different types - escape_field!(astate, args[i], AliasEscapes[i-1]) + escape_field!(astate, args[i], info) + push!(info, -pc) # record def + # propagate the escape information of this object ignoring field information + add_escape_change!(astate, args[i], ignore_aliasinfo(objinfo)) end else # this object has been used as array, but it is allocated as struct here (i.e. should throw) # update obj's field information and just handle this case conservatively - @assert isa(AliasEscapes, ArrayEscapes) objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - @goto conservative_propagation + @label conservative_propagation + # the fields couldn't be analyzed precisely: propagate the entire escape information + # of this object to all its fields as the most conservative propagation + for i in 2:nargs + add_escape_change!(astate, args[i], objinfo) + end end if !(getinst(astate.ir, pc)[:flag] & IR_FLAG_EFFECT_FREE ≠ 0) add_thrown_escapes!(astate, pc, args) end end -function escape_field!(astate::AnalysisState, @nospecialize(v), xf::FieldEscape) +function escape_field!(astate::AnalysisState, @nospecialize(v), xf::AInfo) estate = astate.estate for xidx in xf - x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape + xidx < 0 && continue # ignore def + x = SSAValue(xidx) # obviously this won't be true once we implement ArgEscape add_alias_change!(astate, v, x) end end function escape_unanalyzable_obj!(astate::AnalysisState, @nospecialize(obj), objinfo::EscapeLattice) - objinfo = EscapeLattice(objinfo, TOP_ALIAS_ESCAPES) + objinfo = EscapeLattice(objinfo, TOP_ALIAS_INFO) add_escape_change!(astate, obj, objinfo) return objinfo end @@ -952,9 +970,13 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) nargs = length(args) if nargs < 6 # invalid foreigncall, just escape everything - return add_thrown_escapes!(astate, pc, args) + for i = 1:length(args) + add_escape_change!(astate, args[i], ⊤) + end + return end - foreigncall_nargs = length((args[3])::SimpleVector) + argtypes = args[3]::SimpleVector + nargs = length(argtypes) name = args[1] nn = normalize(name) if isa(nn, Symbol) @@ -976,21 +998,35 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) # end end # NOTE array allocations might have been proven as nothrow (https://github.com/JuliaLang/julia/pull/43565) - info = astate.ir.stmts[pc][:flag] & IR_FLAG_EFFECT_FREE ≠ 0 ? - EscapeLattice(NoEscape(), #=AliasEscapes=#true) : - EscapeLattice(ThrownEscape(pc), #=AliasEscapes=#true) - add_escape_change!(astate, name, info) - for i in 6:5+foreigncall_nargs - add_escape_change!(astate, args[i], info) + nothrow = astate.ir.stmts[pc][:flag] & IR_FLAG_EFFECT_FREE ≠ 0 + if nothrow + name_info = NoEscape() + else + name_info = ThrownEscape(pc) + end + add_escape_change!(astate, name, name_info) + for i = 1:nargs + # we should escape this argument if it is directly called, + # otherwise just impose ThrownEscape if not nothrow + if argtypes[i] === Any + arg_info = ⊤ + else + if nothrow + arg_info = NoEscape() + else + arg_info = ThrownEscape(pc) + end + end + add_escape_change!(astate, args[5+i], arg_info) + end + preserve_info = NoEscape() # TODO encode liveness + for i = (5+nargs):length(args) + add_escape_change!(astate, args[i], preserve_info) end end normalize(@nospecialize x) = isa(x, QuoteNode) ? x.value : x -# NOTE error cases will be handled in `analyze_escapes` anyway, so we don't need to take care of them below -# TODO implement more builtins, make them more accurate -# TODO use `T_IFUNC`-like logic and don't not abuse dispatch ? - function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) ir = astate.ir ft = argextype(first(args), ir, ir.sptypes, ir.argtypes) @@ -1004,7 +1040,7 @@ function escape_call!(astate::AnalysisState, pc::Int, args::Vector{Any}) push!(argtypes, isexpr(arg, :call) ? Any : argextype(arg, ir)) end intrinsic_nothrow(f, argtypes) || add_thrown_escapes!(astate, pc, args, 2) - return # TODO accounts for pointer operations + return # TODO accounts for pointer operations? end result = escape_builtin!(f, astate, pc, args) if result === missing @@ -1067,6 +1103,47 @@ function escape_builtin!(::typeof(tuple), astate::AnalysisState, pc::Int, args:: return false end +function analyze_fields(ir::IRCode, @nospecialize(typ), @nospecialize(fld)) + nfields = fieldcount_noerror(typ) + if nfields === nothing + return Unindexable(false, AInfo()), 0 + end + if isa(typ, DataType) + fldval = try_compute_field(ir, fld) + fidx = try_compute_fieldidx(typ, fldval) + else + fidx = nothing + end + if fidx === nothing + return Unindexable(false, AInfo()), 0 + end + return Indexable(false, AInfo[AInfo() for _ in 1:nfields]), fidx +end + +function reanalyze_fields(ir::IRCode, AliasInfo::Indexable, @nospecialize(typ), @nospecialize(fld)) + infos = AliasInfo.infos + nfields = fieldcount_noerror(typ) + if nfields === nothing + return Unindexable(false, merge_to_unindexable(infos)), 0 + end + if isa(typ, DataType) + fldval = try_compute_field(ir, fld) + fidx = try_compute_fieldidx(typ, fldval) + else + fidx = nothing + end + if fidx === nothing + return Unindexable(false, merge_to_unindexable(infos)), 0 + end + ninfos = length(infos) + if nfields > ninfos + for _ in 1:(nfields-ninfos) + push!(infos, AInfo()) + end + end + return AliasInfo, fidx +end + function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, args::Vector{Any}) length(args) ≥ 3 || return false ir, estate = astate.ir, astate.estate @@ -1080,64 +1157,38 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg else return false end - AliasEscapes = objinfo.AliasEscapes - if isa(AliasEscapes, Bool) - if !AliasEscapes - # the fields of this object haven't been analyzed yet: analyze them now - nfields = fieldcount_noerror(typ) - if nfields !== nothing - AliasEscapes = FieldEscape[FieldEscape() for _ in 1:nfields] - @goto record_field_escape - end - # unsuccessful field analysis: update obj's field information - objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) + AliasInfo = objinfo.AliasInfo + if isa(AliasInfo, Bool) + AliasInfo && @goto conservative_propagation + # the fields of this object haven't been analyzed yet: analyze them now + AliasInfo, fidx = analyze_fields(ir, typ, args[3]) + if isa(AliasInfo, Indexable) + @goto record_indexable_use + else + @goto record_unindexable_use end + elseif isa(AliasInfo, Indexable) && !AliasInfo.array + AliasInfo, fidx = reanalyze_fields(ir, AliasInfo, typ, args[3]) + isa(AliasInfo, Unindexable) && @goto record_unindexable_use + @label record_indexable_use + push!(AliasInfo.infos[fidx], pc) # record use + objinfo = EscapeLattice(objinfo, AliasInfo) + add_escape_change!(astate, obj, objinfo) + elseif isa(AliasInfo, Unindexable) && !AliasInfo.array + @label record_unindexable_use + push!(AliasInfo.info, pc) # record use + objinfo = EscapeLattice(objinfo, AliasInfo) + add_escape_change!(astate, obj, objinfo) + else + # this object has been used as array, but it is used as struct here (i.e. should throw) + # update obj's field information and just handle this case conservatively + objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) @label conservative_propagation # the field couldn't be analyzed precisely: propagate the escape information # imposed on the return value of this `getfield` call to the object itself # as the most conservative propagation ssainfo = estate[SSAValue(pc)] add_escape_change!(astate, obj, ssainfo) - elseif isa(AliasEscapes, FieldEscapes) - nfields = fieldcount_noerror(typ) - if nfields === nothing - # unsuccessful field analysis: update obj's field information - objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - @goto conservative_propagation - else - AliasEscapes = copy(AliasEscapes) - if nfields > length(AliasEscapes) - for _ in 1:(nfields-length(AliasEscapes)) - push!(AliasEscapes, FieldEscape()) - end - end - end - # fields are known: record the return value of this `getfield` call as a possibility - # that imposes escape on field(s) being referenced - @label record_field_escape - if isa(typ, DataType) - fld = args[3] - fldval = try_compute_field(ir, fld) - fidx = try_compute_fieldidx(typ, fldval) - else - fidx = nothing - end - if fidx !== nothing - # the field is known precisely: propagate this escape information to the field - push!(AliasEscapes[fidx], iridx(SSAValue(pc), estate)) - else - # the field isn't known precisely: propagate this escape information to all the fields - for FieldEscape in AliasEscapes - push!(FieldEscape, iridx(SSAValue(pc), estate)) - end - end - add_escape_change!(astate, obj, EscapeLattice(objinfo, AliasEscapes)) - else - # this object has been used as array, but it is used as struct here (i.e. should throw) - # update obj's field information and just handle this case conservatively - @assert isa(AliasEscapes, ArrayEscapes) - objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - @goto conservative_propagation end return false end @@ -1154,65 +1205,45 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar add_escape_change!(astate, val, ⊤) @goto add_thrown_escapes end - AliasEscapes = objinfo.AliasEscapes - if isa(AliasEscapes, Bool) - if !AliasEscapes - # the fields of this object haven't been analyzed yet: analyze them now - typ = widenconst(argextype(obj, ir)) - nfields = fieldcount_noerror(typ) - if nfields !== nothing - # successful field analysis: update obj's field information - AliasEscapes = FieldEscape[FieldEscape() for _ in 1:nfields] - objinfo = EscapeLattice(objinfo, AliasEscapes) - add_escape_change!(astate, obj, objinfo) - @goto add_field_escape - end - # unsuccessful field analysis: update obj's field information - objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - end - @label conservative_propagation - # the field couldn't be analyzed precisely: propagate the entire escape information - # of this object to the value being assigned as the most conservative propagation - add_escape_change!(astate, val, objinfo) - elseif isa(AliasEscapes, FieldEscapes) + AliasInfo = objinfo.AliasInfo + if isa(AliasInfo, Bool) + AliasInfo && @goto conservative_propagation + # the fields of this object haven't been analyzed yet: analyze them now typ = widenconst(argextype(obj, ir)) - nfields = fieldcount_noerror(typ) - if nfields === nothing - # unsuccessful field analysis: update obj's field information - objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - @goto conservative_propagation - elseif nfields > length(AliasEscapes) - AliasEscapes = copy(AliasEscapes) - for _ in 1:(nfields-length(AliasEscapes)) - push!(AliasEscapes, FieldEscape()) - end - end - # fields are known: propagate escape information imposed on recorded possibilities - @label add_field_escape - if isa(typ, DataType) - fld = args[3] - fldval = try_compute_field(ir, fld) - fidx = try_compute_fieldidx(typ, fldval) + AliasInfo, fidx = analyze_fields(ir, typ, args[3]) + if isa(AliasInfo, Indexable) + @goto escape_indexable_def else - fidx = nothing + @goto escape_unindexable_def end - if fidx !== nothing - # the field is known precisely: propagate this escape information to the field - escape_field!(astate, val, AliasEscapes[fidx]) - else - # the field isn't known precisely: propagate this escape information to all the fields - for FieldEscape in AliasEscapes - escape_field!(astate, val, FieldEscape) - end - end - # fields are known: propagate the escape information of this object ignoring field information - add_escape_change!(astate, val, ignore_aliasescapes(objinfo)) + elseif isa(AliasInfo, Indexable) && !AliasInfo.array + typ = widenconst(argextype(obj, ir)) + AliasInfo, fidx = reanalyze_fields(ir, AliasInfo, typ, args[3]) + isa(AliasInfo, Unindexable) && @goto escape_unindexable_def + @label escape_indexable_def + escape_field!(astate, val, AliasInfo.infos[fidx]) + push!(AliasInfo.infos[fidx], -pc) # record def + objinfo = EscapeLattice(objinfo, AliasInfo) + add_escape_change!(astate, obj, objinfo) + # propagate the escape information of this object ignoring field information + add_escape_change!(astate, val, ignore_aliasinfo(objinfo)) + elseif isa(AliasInfo, Unindexable) && !AliasInfo.array + info = AliasInfo.info + @label escape_unindexable_def + escape_field!(astate, val, AliasInfo.info) + push!(AliasInfo.info, -pc) # record def + objinfo = EscapeLattice(objinfo, AliasInfo) + add_escape_change!(astate, obj, objinfo) + # propagate the escape information of this object ignoring field information + add_escape_change!(astate, val, ignore_aliasinfo(objinfo)) else # this object has been used as array, but it is "used" as struct here (i.e. should throw) # update obj's field information and just handle this case conservatively - @assert isa(AliasEscapes, ArrayEscapes) objinfo = escape_unanalyzable_obj!(astate, obj, objinfo) - @goto conservative_propagation + @label conservative_propagation + # the field couldn't be analyzed: propagate the entire escape information + # of this object to the value being assigned as the most conservative propagation + add_escape_change!(astate, val, objinfo) end # also propagate escape information imposed on the return value of this `setfield!` ssainfo = estate[SSAValue(pc)] @@ -1252,29 +1283,26 @@ function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, arg else return true end - AliasEscapes = aryinfo.AliasEscapes - ret = SSAValue(pc) - if isa(AliasEscapes, Bool) - if !AliasEscapes - # the elements of this array haven't been analyzed yet: set ArrayEscapes now - AliasEscapes = ArrayEscapes() - @goto record_element_escape - end - @label conservative_propagation - ssainfo = estate[ret] - add_escape_change!(astate, ary, ssainfo) - elseif isa(AliasEscapes, ArrayEscapes) + AliasInfo = aryinfo.AliasInfo + if isa(AliasInfo, Bool) + AliasInfo && @goto conservative_propagation + # the elements of this array haven't been analyzed yet: set AliasInfo now + AliasInfo = Unindexable(true, AInfo()) + @goto record_unindexable_use + elseif isa(AliasInfo, Indexable) && AliasInfo.array + throw("array index analysis unsupported") + elseif isa(AliasInfo, Unindexable) && AliasInfo.array # record the return value of this `arrayref` call as a possibility that imposes escape - AliasEscapes = copy(AliasEscapes) - @label record_element_escape - push!(AliasEscapes, iridx(ret, estate)) - add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasEscapes)) + @label record_unindexable_use + push!(AliasInfo.info, pc) # record use + add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasInfo)) else # this object has been used as struct, but it is used as array here (thus should throw) # update ary's element information and just handle this case conservatively - @assert isa(AliasEscapes, FieldEscapes) aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) - @goto conservative_propagation + @label conservative_propagation + ssainfo = estate[SSAValue(pc)] + add_escape_change!(astate, ary, ssainfo) end return true end @@ -1308,24 +1336,27 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg add_escape_change!(astate, val, ⊤) return true end - AliasEscapes = aryinfo.AliasEscapes - if isa(AliasEscapes, Bool) - if !AliasEscapes - # the elements of this array haven't been analyzed yet: don't need to consider ArrayEscapes for now - @goto add_ary_escape - end - @label conservative_propagation - add_escape_change!(astate, val, aryinfo) - elseif isa(AliasEscapes, ArrayEscapes) - escape_elements!(astate, val, AliasEscapes) - @label add_ary_escape - add_escape_change!(astate, val, ignore_aliasescapes(aryinfo)) + AliasInfo = aryinfo.AliasInfo + if isa(AliasInfo, Bool) + AliasInfo && @goto conservative_propagation + # the elements of this array haven't been analyzed yet: set AliasInfo now + AliasInfo = Unindexable(true, AInfo()) + @goto escape_unindexable_def + elseif isa(AliasInfo, Indexable) && AliasInfo.array + throw("array index analysis unsupported") + elseif isa(AliasInfo, Unindexable) && AliasInfo.array + @label escape_unindexable_def + escape_elements!(astate, val, AliasInfo.info) + push!(AliasInfo.info, -pc) # record def + add_escape_change!(astate, ary, EscapeLattice(aryinfo, AliasInfo)) + # propagate the escape information of this array ignoring elements information + add_escape_change!(astate, val, ignore_aliasinfo(aryinfo)) else # this object has been used as struct, but it is "used" as array here (thus should throw) # update ary's element information and just handle this case conservatively - @assert isa(AliasEscapes, FieldEscapes) aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) - @goto conservative_propagation + @label conservative_propagation + add_escape_change!(astate, val, aryinfo) end # also propagate escape information imposed on the return value of this `arrayset` ssainfo = estate[SSAValue(pc)] @@ -1333,10 +1364,11 @@ function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, arg return true end -function escape_elements!(astate::AnalysisState, @nospecialize(v), xa::ArrayEscapes) +function escape_elements!(astate::AnalysisState, @nospecialize(v), info::AInfo) estate = astate.estate - for xidx in xa - x = irval(xidx, estate)::SSAValue # TODO remove me once we implement ArgEscape + for xidx in info + xidx < 0 && continue # ignore def + x = SSAValue(xidx) # obviously this won't be true once we implement ArgEscape add_alias_change!(astate, v, x) end end From bca548fcfe1c0b8cd177759ca2e49cc52a904530 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Thu, 27 Jan 2022 18:48:38 -0500 Subject: [PATCH 40/41] Remove maybecopy code --- base/array.jl | 9 ----- base/boot.jl | 21 +++--------- test/compiler/immutablearray.jl | 58 +-------------------------------- 3 files changed, 5 insertions(+), 83 deletions(-) diff --git a/base/array.jl b/base/array.jl index 696a54e795aba..81a2187707552 100644 --- a/base/array.jl +++ b/base/array.jl @@ -415,15 +415,6 @@ similar(a::Array{T}, m::Int) where {T} = Vector{T}(undef, m) similar(a::Array, T::Type, dims::Dims{N}) where {N} = Array{T,N}(undef, dims) similar(a::Array{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) -""" - maybecopy(x) - -`maybecopy` provides access to `x` while ensuring it does not escape. -To do so, the optimizer decides whether to create a copy of `x` or not based on the implementation -That is, `maybecopy` will either be a call to [`copy`](@ref) or just a reference to x. -""" -const maybecopy = Core.maybecopy - # T[x...] constructs Array{T,1} """ getindex(type[, elements...]) diff --git a/base/boot.jl b/base/boot.jl index 00b6af505d295..1469e4d5c8e69 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -274,14 +274,8 @@ struct BoundsError <: Exception a::Any i::Any BoundsError() = new() - # maybecopy --- non-semantic copy - # if escape analysis proves that this throw is the only place where an object would escape local scope, - # creates a copy to avoid that escape and enable memory optimization through memory_opt! - # otherwise if there are other escapes, maybecopy does not copy and just passes the object - BoundsError(@nospecialize(a)) = (@noinline; - a isa Array ? new(Core.maybecopy(a)) : new(a)) - BoundsError(@nospecialize(a), i) = (@noinline; - a isa Array ? new(Core.maybecopy(a), i) : new(a, i)) + BoundsError(@nospecialize(a)) = (@noinline; new(a)) + BoundsError(@nospecialize(a), i) = (@noinline; new(a,i)) end struct DivideError <: Exception end struct OutOfMemoryError <: Exception end @@ -471,10 +465,11 @@ Array{T,N}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = ccall(:jl_new_arr Array{T}(::UndefInitializer, m::Int) where {T} = Array{T,1}(undef, m) Array{T}(::UndefInitializer, m::Int, n::Int) where {T} = Array{T,2}(undef, m, n) Array{T}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = Array{T,3}(undef, m, n, o) -Array{T}(::UndefInitializer, d::NTuple{N,Int}#=::Dims=#) where {T,N} = Array{T,N}(undef, d) +Array{T}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = Array{T,N}(undef, d) # empty vector constructor Array{T,1}() where {T} = Array{T,1}(undef, 0) + (Array{T,N} where T)(x::AbstractArray{S,N}) where {S,N} = Array{S,N}(x) Array(A::AbstractArray{T,N}) where {T,N} = Array{T,N}(A) @@ -482,14 +477,6 @@ Array{T}(A::AbstractArray{S,N}) where {T,N,S} = Array{T,N}(A) AbstractArray{T}(A::AbstractArray{S,N}) where {T,S,N} = AbstractArray{T,N}(A) -# freeze and thaw constructors -ImmutableArray(a::Array) = arrayfreeze(a) -ImmutableArray(a::AbstractArray{T,N}) where {T,N} = ImmutableArray{T,N}(a) -Array(a::ImmutableArray) = arraythaw(a) -# undef initializers -ImmutableArray{T,N}(::UndefInitializer, args...) where {T,N} = ImmutableArray(Array{T,N}(undef, args...)) -ImmutableArray{T}(::UndefInitializer, args...) where {T} = ImmutableArray(Array{T}(undef, args...)) - # primitive Symbol constructors eval(Core, :(function Symbol(s::String) $(Expr(:meta, :pure)) diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl index 520d82de80844..9beb24df1dc41 100644 --- a/test/compiler/immutablearray.jl +++ b/test/compiler/immutablearray.jl @@ -419,60 +419,4 @@ let # escapes via BoundsError ia = unoptimizable(ImmutableArray) @test g[] !== ia end -end - -# @testset "maybecopy tests" begin -# g = nothing # global - -# @noinline function escape(arr) -# g = arr -# return arr -# end - -# function mc1() -# a = Vector{Int64}(undef, 5) -# b = Core.maybecopy(a) # doesn't escape in this function - so a === b -# @test a === b -# end - -# # XXX broken until maybecopy implementation is correct -# function mc2() -# a = Vector{Int64}(undef, 5) -# try -# getindex(a, 6) -# catch e -# if isa(e, BoundsError) -# @test_broken !(e.a === a) # only escapes through throw, so this should copy -# end -# end -# end - -# function mc3() -# a = Vector{Int64}(undef, 5) -# escape(a) -# b = Core.maybecopy(a) -# @test a === b # escapes elsewhere, so give back the actual object -# end - -# function mc4() -# a = Vector{Int64}(undef, 5) -# escape(a) -# try -# getindex(a, 6) -# catch e -# if isa(e, BoundsError) -# @test e.a === a # already unoptimizable_ so we don't copy -# end -# end -# end - -# function test_maybecopy() -# mc1(); mc2(); mc3(); -# mc4(); -# end - -# test_maybecopy() -# end - -# Check that broadcast precedence is working correctly -@test typeof(ImmutableArray([1,2,3]) .+ ImmutableArray([4,5,6])) <: ImmutableArray +end \ No newline at end of file From df8bccc078a8e7caf508995dc01567dca1dd342a Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Thu, 27 Jan 2022 18:52:13 -0500 Subject: [PATCH 41/41] Add tests --- test/immutablearray.jl | 155 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 141 insertions(+), 14 deletions(-) diff --git a/test/immutablearray.jl b/test/immutablearray.jl index c7fce37cd8a6f..dcff145b263a5 100644 --- a/test/immutablearray.jl +++ b/test/immutablearray.jl @@ -13,33 +13,29 @@ import Core: arrayfreeze, mutating_arrayfreeze, arraythaw getindex(a, i) == getindex(b, i) end @test size(a) == size(b) - if t in (Float16, Float32, Float64) - # @test_broken sum(a) == sum(b) # issue #43772, sometimes works, sometimes doesn't - else + if !(t in (Float16, Float32, Float64)) @test sum(a) == sum(b) end @test reverse(a) == reverse(b) @test ndims(a) == ndims(b) - @test axes(a) == axes(b) + for d in 1:ndims(a) + @test axes(a, d) == axes(b, d) + end @test strides(a) == strides(b) @test keys(a) == keys(b) - @test IndexStyle(a) == IndexStyle(b) # ImmutableArray is IndexCartesian whereas Array is IndexLinear - worth looking into + @test IndexStyle(a) == IndexStyle(b) @test eachindex(a) == eachindex(b) + @test isempty(a) == isempty(b) + # Check that broadcast precedence is working correctly + @test typeof(a .+ b) <: ImmutableArray end + end @testset "ImmutableArray builtins" begin - # basic functionality - let - a = [1,2,3] - b = ImmutableArray(a) - @test arrayfreeze(a) === b - @test mutating_arrayfreeze(a) === b - @test arraythaw(b) !== a # arraythaw copies so not === - end - # errors a = [1,2,3] b = ImmutableArray(a) + # errors @test_throws ArgumentError arrayfreeze() @test_throws ArgumentError arrayfreeze([1,2,3], nothing) @test_throws TypeError arrayfreeze(b) @@ -52,4 +48,135 @@ end @test_throws ArgumentError arraythaw([1,2,3], nothing) @test_throws TypeError arraythaw(a) @test_throws TypeError arraythaw("not an array") + + @test arrayfreeze(a) === b + @test arraythaw(b) !== a # arraythaw copies so not === + @test arraythaw(arrayfreeze(a)) == a + @test arraythaw(arrayfreeze(a)) !== a + @test arrayfreeze(arraythaw(b)) === b + @test arraythaw(arrayfreeze(arraythaw(b))) == b + @test arraythaw(arrayfreeze(arraythaw(b))) !== b + + mutating_arrayfreeze(a) # last because this mutates a + @test isa(a, ImmutableArray) + @test a === b + @test arraythaw(a) !== a + @test !isa(arraythaw(a), ImmutableArray) +end + +A = ImmutableArray(rand(5,4,3)) +@testset "Bounds checking" begin + @test checkbounds(Bool, A, 1, 1, 1) == true + @test checkbounds(Bool, A, 5, 4, 3) == true + @test checkbounds(Bool, A, 0, 1, 1) == false + @test checkbounds(Bool, A, 1, 0, 1) == false + @test checkbounds(Bool, A, 1, 1, 0) == false + @test checkbounds(Bool, A, 6, 4, 3) == false + @test checkbounds(Bool, A, 5, 5, 3) == false + @test checkbounds(Bool, A, 5, 4, 4) == false + @test checkbounds(Bool, A, 1) == true # linear indexing + @test checkbounds(Bool, A, 60) == true + @test checkbounds(Bool, A, 61) == false + @test checkbounds(Bool, A, 2, 2, 2, 1) == true # extra indices + @test checkbounds(Bool, A, 2, 2, 2, 2) == false + @test checkbounds(Bool, A, 1, 1) == false + @test checkbounds(Bool, A, 1, 12) == false + @test checkbounds(Bool, A, 5, 12) == false + @test checkbounds(Bool, A, 1, 13) == false + @test checkbounds(Bool, A, 6, 12) == false end + +@testset "single CartesianIndex" begin + @test checkbounds(Bool, A, CartesianIndex((1, 1, 1))) == true + @test checkbounds(Bool, A, CartesianIndex((5, 4, 3))) == true + @test checkbounds(Bool, A, CartesianIndex((0, 1, 1))) == false + @test checkbounds(Bool, A, CartesianIndex((1, 0, 1))) == false + @test checkbounds(Bool, A, CartesianIndex((1, 1, 0))) == false + @test checkbounds(Bool, A, CartesianIndex((6, 4, 3))) == false + @test checkbounds(Bool, A, CartesianIndex((5, 5, 3))) == false + @test checkbounds(Bool, A, CartesianIndex((5, 4, 4))) == false + @test checkbounds(Bool, A, CartesianIndex((1,))) == false + @test checkbounds(Bool, A, CartesianIndex((60,))) == false + @test checkbounds(Bool, A, CartesianIndex((61,))) == false + @test checkbounds(Bool, A, CartesianIndex((2, 2, 2, 1,))) == true + @test checkbounds(Bool, A, CartesianIndex((2, 2, 2, 2,))) == false + @test checkbounds(Bool, A, CartesianIndex((1, 1,))) == false + @test checkbounds(Bool, A, CartesianIndex((1, 12,))) == false + @test checkbounds(Bool, A, CartesianIndex((5, 12,))) == false + @test checkbounds(Bool, A, CartesianIndex((1, 13,))) == false + @test checkbounds(Bool, A, CartesianIndex((6, 12,))) == false +end + +@testset "mix of CartesianIndex and Int" begin + @test checkbounds(Bool, A, CartesianIndex((1,)), 1, CartesianIndex((1,))) == true + @test checkbounds(Bool, A, CartesianIndex((5, 4)), 3) == true + @test checkbounds(Bool, A, CartesianIndex((0, 1)), 1) == false + @test checkbounds(Bool, A, 1, CartesianIndex((0, 1))) == false + @test checkbounds(Bool, A, 1, 1, CartesianIndex((0,))) == false + @test checkbounds(Bool, A, 6, CartesianIndex((4, 3))) == false + @test checkbounds(Bool, A, 5, CartesianIndex((5,)), 3) == false + @test checkbounds(Bool, A, CartesianIndex((5,)), CartesianIndex((4,)), CartesianIndex((4,))) == false +end + +@testset "vector indices" begin + @test checkbounds(Bool, A, 1:5, 1:4, 1:3) == true + @test checkbounds(Bool, A, 0:5, 1:4, 1:3) == false + @test checkbounds(Bool, A, 1:5, 0:4, 1:3) == false + @test checkbounds(Bool, A, 1:5, 1:4, 0:3) == false + @test checkbounds(Bool, A, 1:6, 1:4, 1:3) == false + @test checkbounds(Bool, A, 1:5, 1:5, 1:3) == false + @test checkbounds(Bool, A, 1:5, 1:4, 1:4) == false + @test checkbounds(Bool, A, 1:60) == true + @test checkbounds(Bool, A, 1:61) == false + @test checkbounds(Bool, A, 2, 2, 2, 1:1) == true # extra indices + @test checkbounds(Bool, A, 2, 2, 2, 1:2) == false + @test checkbounds(Bool, A, 1:5, 1:4) == false + @test checkbounds(Bool, A, 1:5, 1:12) == false + @test checkbounds(Bool, A, 1:5, 1:13) == false + @test checkbounds(Bool, A, 1:6, 1:12) == false +end + +@testset "logical" begin + @test checkbounds(Bool, A, trues(5), trues(4), trues(3)) == true + @test checkbounds(Bool, A, trues(6), trues(4), trues(3)) == false + @test checkbounds(Bool, A, trues(5), trues(5), trues(3)) == false + @test checkbounds(Bool, A, trues(5), trues(4), trues(4)) == false + @test checkbounds(Bool, A, trues(60)) == true + @test checkbounds(Bool, A, trues(61)) == false + @test checkbounds(Bool, A, 2, 2, 2, trues(1)) == true # extra indices + @test checkbounds(Bool, A, 2, 2, 2, trues(2)) == false + @test checkbounds(Bool, A, trues(5), trues(12)) == false + @test checkbounds(Bool, A, trues(5), trues(13)) == false + @test checkbounds(Bool, A, trues(6), trues(12)) == false + @test checkbounds(Bool, A, trues(5, 4, 3)) == true + @test checkbounds(Bool, A, trues(5, 4, 2)) == false + @test checkbounds(Bool, A, trues(5, 12)) == false + @test checkbounds(Bool, A, trues(1, 5), trues(1, 4, 1), trues(1, 1, 3)) == false + @test checkbounds(Bool, A, trues(1, 5), trues(1, 4, 1), trues(1, 1, 2)) == false + @test checkbounds(Bool, A, trues(1, 5), trues(1, 5, 1), trues(1, 1, 3)) == false + @test checkbounds(Bool, A, trues(1, 5), :, 2) == false + @test checkbounds(Bool, A, trues(5, 4), trues(3)) == true + @test checkbounds(Bool, A, trues(4, 4), trues(3)) == true + @test checkbounds(Bool, A, trues(5, 4), trues(2)) == false + @test checkbounds(Bool, A, trues(6, 4), trues(3)) == false + @test checkbounds(Bool, A, trues(5, 4), trues(4)) == false +end + +@testset "array of CartesianIndex" begin + @test checkbounds(Bool, A, [CartesianIndex((1, 1, 1))]) == true + @test checkbounds(Bool, A, [CartesianIndex((5, 4, 3))]) == true + @test checkbounds(Bool, A, [CartesianIndex((0, 1, 1))]) == false + @test checkbounds(Bool, A, [CartesianIndex((1, 0, 1))]) == false + @test checkbounds(Bool, A, [CartesianIndex((1, 1, 0))]) == false + @test checkbounds(Bool, A, [CartesianIndex((6, 4, 3))]) == false + @test checkbounds(Bool, A, [CartesianIndex((5, 5, 3))]) == false + @test checkbounds(Bool, A, [CartesianIndex((5, 4, 4))]) == false + @test checkbounds(Bool, A, [CartesianIndex((1, 1))], 1) == true + @test checkbounds(Bool, A, [CartesianIndex((5, 4))], 3) == true + @test checkbounds(Bool, A, [CartesianIndex((0, 1))], 1) == false + @test checkbounds(Bool, A, [CartesianIndex((1, 0))], 1) == false + @test checkbounds(Bool, A, [CartesianIndex((1, 1))], 0) == false + @test checkbounds(Bool, A, [CartesianIndex((6, 4))], 3) == false + @test checkbounds(Bool, A, [CartesianIndex((5, 5))], 3) == false + @test checkbounds(Bool, A, [CartesianIndex((5, 4))], 4) == false +end \ No newline at end of file