From df40ca098a6a2828d617dd473f14e9f9bb83b7e3 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 13 Sep 2022 23:33:55 +0900 Subject: [PATCH 1/3] inference: setup separate functions for each backedge kind Also changes the argument list so that they are ordered as `(caller, [backedge information])`. --- base/compiler/abstractinterpretation.jl | 18 ++++++----- base/compiler/inferencestate.jl | 41 ++++++++++++++++--------- base/compiler/optimize.jl | 8 +++-- base/compiler/ssair/inlining.jl | 24 +++++++++++++-- base/compiler/typeinfer.jl | 2 +- 5 files changed, 65 insertions(+), 28 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 7f2106b8f00701..8eb85bc3a509c0 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -483,15 +483,15 @@ function add_call_backedges!(interp::AbstractInterpreter, end end for edge in edges - add_backedge!(edge, sv) + add_backedge!(sv, edge) end # also need an edge to the method table in case something gets # added that did not intersect with any existing method if isa(matches, MethodMatches) - matches.fullmatch || add_mt_backedge!(matches.mt, atype, sv) + matches.fullmatch || add_mt_backedge!(sv, matches.mt, atype) else for (thisfullmatch, mt) in zip(matches.fullmatches, matches.mts) - thisfullmatch || add_mt_backedge!(mt, atype, sv) + thisfullmatch || add_mt_backedge!(sv, mt, atype) end end end @@ -882,7 +882,11 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, resul end res = concrete_eval_call(interp, f, result, arginfo, sv) if isa(res, ConstCallResults) - add_backedge!(res.const_result.mi, sv, invoketypes) + if invoketypes === nothing + add_backedge!(sv, res.const_result.mi) + else + add_invoke_backedge!(sv, invoketypes, res.const_result.mi) + end return res end mi = maybe_get_const_prop_profitable(interp, result, f, arginfo, match, sv) @@ -929,7 +933,7 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, resul result = inf_result.result # if constant inference hits a cycle, just bail out isa(result, InferenceState) && return nothing - add_backedge!(mi, sv) + add_backedge!(sv, mi) return ConstCallResults(result, ConstPropResult(inf_result), inf_result.ipo_effects) end @@ -1685,7 +1689,7 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn ti = tienv[1]; env = tienv[2]::SimpleVector result = abstract_call_method(interp, method, ti, env, false, sv) (; rt, edge, effects) = result - edge !== nothing && add_backedge!(edge::MethodInstance, sv, types) + edge !== nothing && add_invoke_backedge!(sv, types, edge::MethodInstance) match = MethodMatch(ti, env, method, argtype <: method.sig) res = nothing sig = match.spec_types @@ -1839,7 +1843,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, sig = argtypes_to_type(arginfo.argtypes) result = abstract_call_method(interp, closure.source, sig, Core.svec(), false, sv) (; rt, edge, effects) = result - edge !== nothing && add_backedge!(edge, sv) + edge !== nothing && add_backedge!(sv, edge) tt = closure.typ sigT = (unwrap_unionall(tt)::DataType).parameters[1] match = MethodMatch(sig, Core.svec(), closure.source, sig <: rewrap_unionall(sigT, tt)) diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index e9bd7474d265de..e1d20f01042c47 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -478,38 +478,49 @@ function record_ssa_assign!(ssa_id::Int, @nospecialize(new), frame::InferenceSta return nothing end -function add_cycle_backedge!(frame::InferenceState, caller::InferenceState, currpc::Int) +function add_cycle_backedge!(caller::InferenceState, frame::InferenceState, currpc::Int) update_valid_age!(frame, caller) backedge = (caller, currpc) contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge) - add_backedge!(frame.linfo, caller) + add_backedge!(caller, frame.linfo) return frame end # temporarily accumulate our edges to later add as backedges in the callee -function add_backedge!(li::MethodInstance, caller::InferenceState, @nospecialize(invokesig=nothing)) - isa(caller.linfo.def, Method) || return # don't add backedges to toplevel exprs - edges = caller.stmt_edges[caller.currpc] - if edges === nothing - edges = caller.stmt_edges[caller.currpc] = [] +function add_backedge!(caller::InferenceState, li::MethodInstance) + edges = get_stmt_edges!(caller) + if edges !== nothing + push!(edges, li) end - if invokesig !== nothing - push!(edges, invokesig) + return nothing +end + +function add_invoke_backedge!(caller::InferenceState, @nospecialize(invokesig::Type), li::MethodInstance) + edges = get_stmt_edges!(caller) + if edges !== nothing + push!(edges, invokesig, li) end - push!(edges, li) return nothing end # used to temporarily accumulate our no method errors to later add as backedges in the callee method table -function add_mt_backedge!(mt::Core.MethodTable, @nospecialize(typ), caller::InferenceState) - isa(caller.linfo.def, Method) || return # don't add backedges to toplevel exprs +function add_mt_backedge!(caller::InferenceState, mt::Core.MethodTable, @nospecialize(typ)) + edges = get_stmt_edges!(caller) + if edges !== nothing + push!(edges, mt, typ) + end + return nothing +end + +function get_stmt_edges!(caller::InferenceState) + if !isa(caller.linfo.def, Method) + return nothing # don't add backedges to toplevel exprs + end edges = caller.stmt_edges[caller.currpc] if edges === nothing edges = caller.stmt_edges[caller.currpc] = [] end - push!(edges, mt) - push!(edges, typ) - return nothing + return edges end function empty_backedges!(frame::InferenceState, currpc::Int = frame.currpc) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 1b6d19bc152b6a..51cc95a728ff07 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -65,9 +65,13 @@ intersect!(et::EdgeTracker, range::WorldRange) = et.valid_worlds[] = intersect(et.valid_worlds[], range) push!(et::EdgeTracker, mi::MethodInstance) = push!(et.edges, mi) -function add_edge!(et::EdgeTracker, @nospecialize(invokesig), mi::MethodInstance) - invokesig === nothing && return push!(et.edges, mi) +function add_backedge!(et::EdgeTracker, mi::MethodInstance) + push!(et.edges, mi) + return nothing +end +function add_invoke_backedge!(et::EdgeTracker, @nospecialize(invokesig), mi::MethodInstance) push!(et.edges, invokesig, mi) + return nothing end function push!(et::EdgeTracker, ci::CodeInstance) intersect!(et, WorldRange(min_world(li), max_world(li))) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index a18066058aebb5..c2bafcacc93ecb 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -832,7 +832,13 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) inferred_src = match.src if isa(inferred_src, ConstAPI) # use constant calling convention - et !== nothing && add_edge!(et, invokesig, mi) + if et !== nothing + if invokesig === nothing + add_backedge!(et, mi) + else + add_invoke_backedge!(et, invokesig, mi) + end + end return ConstantCase(quoted(inferred_src.val)) else src = inferred_src # ::Union{Nothing,CodeInfo} for NativeInterpreter @@ -843,7 +849,13 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) if code isa CodeInstance if use_const_api(code) # in this case function can be inlined to a constant - et !== nothing && add_edge!(et, invokesig, mi) + if et !== nothing + if invokesig === nothing + add_backedge!(et, mi) + else + add_invoke_backedge!(et, invokesig, mi) + end + end return ConstantCase(quoted(code.rettype_const)) else src = @atomic :monotonic code.inferred @@ -867,7 +879,13 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) src === nothing && return compileable_specialization(et, match, effects; compilesig_invokes=state.params.compilesig_invokes) - et !== nothing && add_edge!(et, invokesig, mi) + if et !== nothing + if invokesig === nothing + add_backedge!(et, mi) + else + add_invoke_backedge!(et, invokesig, mi) + end + end return InliningTodo(mi, retrieve_ir_for_inlining(mi, src), effects) end diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index dad2ba0a7e3b14..c5d19166ad7d0f 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -806,7 +806,7 @@ function merge_call_chain!(interp::AbstractInterpreter, parent::InferenceState, # of recursion. merge_effects!(interp, parent, Effects(EFFECTS_TOTAL; terminates=false)) while true - add_cycle_backedge!(child, parent, parent.currpc) + add_cycle_backedge!(parent, child, parent.currpc) union_caller_cycle!(ancestor, child) merge_effects!(interp, child, Effects(EFFECTS_TOTAL; terminates=false)) child = parent From 226cf0edd77bf7153b927cb31c82a10eab0edcd6 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 14 Sep 2022 00:18:27 +0900 Subject: [PATCH 2/3] inference: fix backedge computation for const-prop'ed callsite With this commit `abstract_call_method_with_const_args` doesn't add backedge but rather returns the backedge to the caller, letting the callers like `abstract_call_gf_by_type` and `abstract_invoke` take the responsibility to add backedge to current context appropriately. As a result, this fixes the backedge calculation for const-prop'ed `invoke` callsite. For example, for the following call graph, ```julia foo(a::Int) = a > 0 ? :int : println(a) foo(a::Integer) = a > 0 ? "integer" : println(a) bar(a::Int) = @invoke foo(a::Integer) ``` Previously we added the wrong backedge `nothing, bar(Int64) from bar(Int64)`: ```julia julia> last(only(code_typed(()->bar(42)))) String julia> let m = only(methods(foo, (UInt,))) @eval Core.Compiler for (sig, caller) in BackedgeIterator($m.specializations[1].backedges) println(sig, ", ", caller) end end Tuple{typeof(Main.foo), Integer}, bar(Int64) from bar(Int64) nothing, bar(Int64) from bar(Int64) ``` but now we only add `invoke`-backedge: ```julia julia> last(only(code_typed(()->bar(42)))) String julia> let m = only(methods(foo, (UInt,))) @eval Core.Compiler for (sig, caller) in BackedgeIterator($m.specializations[1].backedges) println(sig, ", ", caller) end end Tuple{typeof(Main.foo), Integer}, bar(Int64) from bar(Int64) ``` --- base/compiler/abstractinterpretation.jl | 45 +++++++++++-------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 8eb85bc3a509c0..f73724cf3c263e 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -126,7 +126,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), for sig_n in splitsigs result = abstract_call_method(interp, method, sig_n, svec(), multiple_matches, sv) (; rt, edge, effects) = result - edge === nothing || push!(edges, edge) this_argtypes = isa(matches, MethodMatches) ? argtypes : matches.applicable_argtypes[i] this_arginfo = ArgInfo(fargs, this_argtypes) const_call_result = abstract_call_method_with_const_args(interp, result, @@ -135,12 +134,13 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if const_call_result !== nothing if const_call_result.rt ⊑ᵢ rt rt = const_call_result.rt - (; effects, const_result) = const_call_result + (; effects, const_result, edge) = const_call_result end end all_effects = merge_effects(all_effects, effects) push!(const_results, const_result) any_const_result |= const_result !== nothing + edge === nothing || push!(edges, edge) this_rt = tmerge(this_rt, rt) if bail_out_call(interp, this_rt, sv) break @@ -153,7 +153,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), (; rt, edge, effects) = result this_conditional = ignorelimited(rt) this_rt = widenwrappedconditional(rt) - edge === nothing || push!(edges, edge) # try constant propagation with argtypes for this match # this is in preparation for inlining, or improving the return result this_argtypes = isa(matches, MethodMatches) ? argtypes : matches.applicable_argtypes[i] @@ -169,12 +168,13 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if this_const_rt ⊑ᵢ this_rt this_conditional = this_const_conditional this_rt = this_const_rt - (; effects, const_result) = const_call_result + (; effects, const_result, edge) = const_call_result end end all_effects = merge_effects(all_effects, effects) push!(const_results, const_result) any_const_result |= const_result !== nothing + edge === nothing || push!(edges, edge) end @assert !(this_conditional isa Conditional) "invalid lattice element returned from inter-procedural context" seen += 1 @@ -831,17 +831,18 @@ function concrete_eval_call(interp::AbstractInterpreter, if eligible args = collect_const_args(arginfo, #=start=#2) world = get_world_counter(interp) + edge = result.edge::MethodInstance value = try Core._call_in_world_total(world, f, args...) catch # The evaluation threw. By :consistent-cy, we're guaranteed this would have happened at runtime - return ConstCallResults(Union{}, ConcreteResult(result.edge::MethodInstance, result.effects), result.effects) + return ConstCallResults(Union{}, ConcreteResult(edge, result.effects), result.effects, edge) end if is_inlineable_constant(value) || call_result_unused(sv) # If the constant is not inlineable, still do the const-prop, since the # code that led to the creation of the Const may be inlineable in the same # circumstance and may be optimizable. - return ConstCallResults(Const(value), ConcreteResult(result.edge::MethodInstance, EFFECTS_TOTAL, value), EFFECTS_TOTAL) + return ConstCallResults(Const(value), ConcreteResult(edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL, edge) end return false else # eligible for semi-concrete evaluation @@ -868,27 +869,22 @@ struct ConstCallResults rt::Any const_result::ConstResult effects::Effects + edge::MethodInstance ConstCallResults(@nospecialize(rt), const_result::ConstResult, - effects::Effects) = - new(rt, const_result, effects) + effects::Effects, + edge::MethodInstance) = + new(rt, const_result, effects, edge) end function abstract_call_method_with_const_args(interp::AbstractInterpreter, result::MethodCallResult, @nospecialize(f), arginfo::ArgInfo, match::MethodMatch, - sv::InferenceState, @nospecialize(invoketypes=nothing)) + sv::InferenceState) if !const_prop_enabled(interp, sv, match) return nothing end res = concrete_eval_call(interp, f, result, arginfo, sv) - if isa(res, ConstCallResults) - if invoketypes === nothing - add_backedge!(sv, res.const_result.mi) - else - add_invoke_backedge!(sv, invoketypes, res.const_result.mi) - end - return res - end + isa(res, ConstCallResults) && return res mi = maybe_get_const_prop_profitable(interp, result, f, arginfo, match, sv) mi === nothing && return nothing # try semi-concrete evaluation @@ -900,7 +896,7 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, resul if isa(ir, IRCode) T = ir_abstract_constant_propagation(interp, mi_cache, sv, mi, ir, arginfo.argtypes) if !isa(T, Type) || typeintersect(T, Bool) === Union{} - return ConstCallResults(T, SemiConcreteResult(mi, ir, result.effects), result.effects) + return ConstCallResults(T, SemiConcreteResult(mi, ir, result.effects), result.effects, mi) end end end @@ -933,8 +929,7 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, resul result = inf_result.result # if constant inference hits a cycle, just bail out isa(result, InferenceState) && return nothing - add_backedge!(sv, mi) - return ConstCallResults(result, ConstPropResult(inf_result), inf_result.ipo_effects) + return ConstCallResults(result, ConstPropResult(inf_result), inf_result.ipo_effects, mi) end # if there's a possibility we could get a better result with these constant arguments @@ -1689,7 +1684,6 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn ti = tienv[1]; env = tienv[2]::SimpleVector result = abstract_call_method(interp, method, ti, env, false, sv) (; rt, edge, effects) = result - edge !== nothing && add_invoke_backedge!(sv, types, edge::MethodInstance) match = MethodMatch(ti, env, method, argtype <: method.sig) res = nothing sig = match.spec_types @@ -1702,14 +1696,15 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn # argtypes′[i] = t ⊑ a ? t : a # end const_call_result = abstract_call_method_with_const_args(interp, result, - overlayed ? nothing : singleton_type(ft′), arginfo, match, sv, types) + overlayed ? nothing : singleton_type(ft′), arginfo, match, sv) const_result = nothing if const_call_result !== nothing if ⊑(typeinf_lattice(interp), const_call_result.rt, rt) - (; rt, effects, const_result) = const_call_result + (; rt, effects, const_result, edge) = const_call_result end end effects = Effects(effects; nonoverlayed=!overlayed) + edge !== nothing && add_invoke_backedge!(sv, types, edge) return CallMeta(from_interprocedural!(ipo_lattice(interp), rt, sv, arginfo, sig), effects, InvokeCallInfo(match, const_result)) end @@ -1843,7 +1838,6 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, sig = argtypes_to_type(arginfo.argtypes) result = abstract_call_method(interp, closure.source, sig, Core.svec(), false, sv) (; rt, edge, effects) = result - edge !== nothing && add_backedge!(sv, edge) tt = closure.typ sigT = (unwrap_unionall(tt)::DataType).parameters[1] match = MethodMatch(sig, Core.svec(), closure.source, sig <: rewrap_unionall(sigT, tt)) @@ -1853,7 +1847,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, nothing, arginfo, match, sv) if const_call_result !== nothing if const_call_result.rt ⊑ rt - (; rt, effects, const_result) = const_call_result + (; rt, effects, const_result, edge) = const_call_result end end end @@ -1869,6 +1863,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, end end rt = from_interprocedural!(ipo, rt, sv, arginfo, match.spec_types) + edge !== nothing && add_backedge!(sv, edge) return CallMeta(rt, effects, info) end From 0f9eb79e228381d1edb48af52d194df4e5ec2b39 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 14 Sep 2022 00:43:36 +0900 Subject: [PATCH 3/3] inference: make `BackedgePair` struct --- base/compiler/typeinfer.jl | 12 ++++++------ base/compiler/utilities.jl | 15 +++++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index c5d19166ad7d0f..6db3c42a6ca54a 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -567,13 +567,13 @@ function store_backedges(frame::InferenceResult, edges::Vector{Any}) nothing end -function store_backedges(caller::MethodInstance, edges::Vector{Any}) - for (typ, to) in BackedgeIterator(edges) - if isa(to, MethodInstance) - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), to, typ, caller) +function store_backedges(frame::MethodInstance, edges::Vector{Any}) + for (; sig, caller) in BackedgeIterator(edges) + if isa(caller, MethodInstance) + ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), caller, sig, frame) else - typeassert(to, Core.MethodTable) - ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), to, typ, caller) + typeassert(caller, Core.MethodTable) + ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), caller, sig, frame) end end end diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index 071b0b6089b988..88e002a469575d 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -235,9 +235,9 @@ is_no_constprop(method::Union{Method,CodeInfo}) = method.constprop == 0x02 Return an iterator over a list of backedges. Iteration returns `(sig, caller)` elements, which will be one of the following: -- `(nothing, caller::MethodInstance)`: a call made by ordinary inferrable dispatch -- `(invokesig, caller::MethodInstance)`: a call made by `invoke(f, invokesig, args...)` -- `(specsig, mt::MethodTable)`: an abstract call +- `BackedgePair(nothing, caller::MethodInstance)`: a call made by ordinary inferrable dispatch +- `BackedgePair(invokesig, caller::MethodInstance)`: a call made by `invoke(f, invokesig, args...)` +- `BackedgePair(specsig, mt::MethodTable)`: an abstract call # Examples @@ -254,7 +254,7 @@ julia> callyou(2.0) julia> mi = first(which(callme, (Any,)).specializations) MethodInstance for callme(::Float64) -julia> @eval Core.Compiler for (sig, caller) in BackedgeIterator(Main.mi.backedges) +julia> @eval Core.Compiler for (; sig, caller) in BackedgeIterator(Main.mi.backedges) println(sig) println(caller) end @@ -268,8 +268,11 @@ end const empty_backedge_iter = BackedgeIterator(Any[]) -const MethodInstanceOrTable = Union{MethodInstance, Core.MethodTable} -const BackedgePair = Pair{Union{Type, Nothing, MethodInstanceOrTable}, MethodInstanceOrTable} +struct BackedgePair + sig # ::Union{Nothing,Type} + caller::Union{MethodInstance,Core.MethodTable} + BackedgePair(@nospecialize(sig), caller::Union{MethodInstance,Core.MethodTable}) = new(sig, caller) +end function iterate(iter::BackedgeIterator, i::Int=1) backedges = iter.backedges