From ff2096312f1c066a0d4047e756f4cf6ef6c771a4 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Mon, 8 Jul 2019 16:36:26 -0400 Subject: [PATCH 1/7] Export jl_resolve_globals_in_ir in Julia src --- src/julia_internal.h | 4 ++++ src/method.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index b9ce810262703..f3e2f9bf0c4ff 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1016,6 +1016,10 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, int isabspath(const char *in); +// TODO(NHDALY): Find the right spot for this. +JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, + jl_svec_t *sparam_vals, int binding_effects); + extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; extern jl_sym_t *empty_sym; extern jl_sym_t *top_sym; extern jl_sym_t *module_sym; extern jl_sym_t *slot_sym; diff --git a/src/method.c b/src/method.c index 7f9189be0d152..35abef0f4c891 100644 --- a/src/method.c +++ b/src/method.c @@ -191,8 +191,8 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve return expr; } -void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, - int binding_effects) +JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, + jl_svec_t *sparam_vals, int binding_effects) { size_t i, l = jl_array_len(stmts); for (i = 0; i < l; i++) { From 71a356b84462280ef706dceac6c5c2f241a40d40 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Fri, 23 Aug 2019 10:55:08 -0400 Subject: [PATCH 2/7] Export new `jl_expand_and_resolve`, instead of jl_resolve_globals_in_ir. Instead of exporting the internal function jl_resolve_globals_in_ir, this factors out the code from generated functions into `jl_expand_and_resolve`, which lowers an expression into a CodeInfo and then resolves globals in the codeinfo in light of provided typenames. I then exposed this to Julia via another method for `Meta.lower()`, this one taking type parameters. --- base/meta.jl | 2 ++ src/ast.c | 1 + src/julia_internal.h | 4 ---- src/method.c | 35 +++++++++++++++++++++++------------ 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/base/meta.jl b/base/meta.jl index 7a85594249301..740f7fa3288c3 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -121,6 +121,8 @@ for executing in module `m`. See also [`code_lowered`](@ref). """ lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m) +lower(m::Module, @nospecialize(x), types::AbstractArray) = + ccall(:jl_expand_and_resolve, Any, (Any, Any, Core.SimpleVector), x, m, Core.svec((Symbol(t) for t in types)...)) """ @lower [m] x diff --git a/src/ast.c b/src/ast.c index fe7615f540a80..4365b0c1431ef 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1183,6 +1183,7 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmod return expr; } +// Lower an expression tree into Julia's intermediate-representation. JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule) { return jl_expand_with_loc(expr, inmodule, "none", 0); diff --git a/src/julia_internal.h b/src/julia_internal.h index f3e2f9bf0c4ff..b9ce810262703 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1016,10 +1016,6 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, int isabspath(const char *in); -// TODO(NHDALY): Find the right spot for this. -JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, - jl_svec_t *sparam_vals, int binding_effects); - extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; extern jl_sym_t *empty_sym; extern jl_sym_t *top_sym; extern jl_sym_t *module_sym; extern jl_sym_t *slot_sym; diff --git a/src/method.c b/src/method.c index 35abef0f4c891..d092b6cc4ce63 100644 --- a/src/method.c +++ b/src/method.c @@ -18,6 +18,8 @@ extern "C" { extern jl_value_t *jl_builtin_getfield; extern jl_value_t *jl_builtin_tuple; +// Resolve references to non-locally-defined variables to become references to global +// variables in `module` (unless the rvalue is one of the type parameters in `sparam_vals`). static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_svec_t *sparam_vals, int binding_effects, int eager_resolve) { @@ -191,8 +193,8 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve return expr; } -JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, - jl_svec_t *sparam_vals, int binding_effects) +void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, + int binding_effects) { size_t i, l = jl_array_len(stmts); for (i = 0; i < l; i++) { @@ -376,7 +378,19 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_method_t *def, jl_value_t *generator return code; } -// return a newly allocated CodeInfo for the function signature +// Lower `ex` into Julia IR, and (if it expands into a CodeInfo) resolve global-variable +// references in light of the provided type parameters. +JL_DLLEXPORT jl_code_info_t *jl_expand_and_resolve(jl_value_t *ex, jl_module_t *module, + jl_svec_t *sparam_vals) { + jl_code_info_t *func = (jl_code_info_t*)jl_expand((jl_value_t*)ex, module); + if (jl_is_code_info(func)) { + jl_array_t *stmts = (jl_array_t*)func->code; + jl_resolve_globals_in_ir(stmts, module, sparam_vals, 1); + } + return func; +} + +// Return a newly allocated CodeInfo for the function signature // effectively described by the tuple (specTypes, env, Method) inside linfo JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo) { @@ -407,17 +421,14 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo) func = (jl_code_info_t*)ex; } else { - func = (jl_code_info_t*)jl_expand((jl_value_t*)ex, def->module); - if (!jl_is_code_info(func)) { - if (jl_is_expr(func) && ((jl_expr_t*)func)->head == error_sym) { - ptls->in_pure_callback = 0; - jl_toplevel_eval(def->module, (jl_value_t*)func); - } + // Lower the user's expression and resolve references to the type parameters + func = jl_expand_and_resolve(ex, def->module, linfo->sparam_vals); + + if (jl_is_expr(func) && ((jl_expr_t*)func)->head == error_sym) { + ptls->in_pure_callback = 0; + jl_toplevel_eval(def->module, (jl_value_t*)func); jl_error("generated function body is not pure. this likely means it contains a closure or comprehension."); } - - jl_array_t *stmts = (jl_array_t*)func->code; - jl_resolve_globals_in_ir(stmts, def->module, linfo->sparam_vals, 1); } ptls->in_pure_callback = last_in; From 3d9feacb6e22c874611360c100d7023efb03fce2 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Mon, 2 Sep 2019 22:15:23 -0400 Subject: [PATCH 3/7] Comment `lower_and_resolve_lambda` --- base/meta.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/base/meta.jl b/base/meta.jl index 740f7fa3288c3..80892dd3471a4 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -121,7 +121,16 @@ for executing in module `m`. See also [`code_lowered`](@ref). """ lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m) -lower(m::Module, @nospecialize(x), types::AbstractArray) = + + +""" + lower_and_resolve_lambda(m, x, types) + +Takes a lambda expression `x` and returns an equivalent expression in lowered form +for executing in module `m`. +See also [`code_lowered`](@ref). +""" +lower_and_resolve_lambda(m::Module, @nospecialize(x), types) = ccall(:jl_expand_and_resolve, Any, (Any, Any, Core.SimpleVector), x, m, Core.svec((Symbol(t) for t in types)...)) """ From 94285d8d2e8b4b671ec200b6290635ac890d16fd Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Thu, 12 Sep 2019 18:16:38 -0400 Subject: [PATCH 4/7] Remove the exposed Julia binding for ccall(:jl_expand_and_resolve), because it's very particular about its inputs --- base/meta.jl | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/base/meta.jl b/base/meta.jl index 80892dd3471a4..7a85594249301 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -122,17 +122,6 @@ See also [`code_lowered`](@ref). """ lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m) - -""" - lower_and_resolve_lambda(m, x, types) - -Takes a lambda expression `x` and returns an equivalent expression in lowered form -for executing in module `m`. -See also [`code_lowered`](@ref). -""" -lower_and_resolve_lambda(m::Module, @nospecialize(x), types) = - ccall(:jl_expand_and_resolve, Any, (Any, Any, Core.SimpleVector), x, m, Core.svec((Symbol(t) for t in types)...)) - """ @lower [m] x From a27b454d16e81c711c999fce7138f84e85ec0e2b Mon Sep 17 00:00:00 2001 From: Nathan Date: Wed, 18 Sep 2019 03:24:17 -0400 Subject: [PATCH 5/7] Restore error states to what it was before this PR: aways error if !is_code_info, maybe without resetting in_pure_callback --- src/method.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/method.c b/src/method.c index e91fd85b625a6..a3fa763dc6f21 100644 --- a/src/method.c +++ b/src/method.c @@ -424,9 +424,11 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo) // Lower the user's expression and resolve references to the type parameters func = jl_expand_and_resolve(ex, def->module, linfo->sparam_vals); - if (jl_is_expr(func) && ((jl_expr_t*)func)->head == error_sym) { - ptls->in_pure_callback = 0; - jl_toplevel_eval(def->module, (jl_value_t*)func); + if (!jl_is_code_info(func)) { + if (jl_is_expr(func) && ((jl_expr_t*)func)->head == error_sym) { + ptls->in_pure_callback = 0; + jl_toplevel_eval(def->module, (jl_value_t*)func); + } jl_error("The function body AST defined by this @generated function is not pure. This likely means it contains a closure or comprehension."); } } From b08231fe1fbf82b8fdd375f5ac20769c6d978fdb Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Wed, 2 Oct 2019 19:03:35 -0400 Subject: [PATCH 6/7] Add error return value semantics to comment on jl_expand_and_resolve --- src/method.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/method.c b/src/method.c index a3fa763dc6f21..41e31b8daeb97 100644 --- a/src/method.c +++ b/src/method.c @@ -380,6 +380,9 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_method_t *def, jl_value_t *generator // Lower `ex` into Julia IR, and (if it expands into a CodeInfo) resolve global-variable // references in light of the provided type parameters. +// Like `jl_expand`, if there is an error expanding the provided expression, the return value +// will be an error expression (an `Expr` with `error_sym` as its head), which should be eval'd +// in the caller's context. JL_DLLEXPORT jl_code_info_t *jl_expand_and_resolve(jl_value_t *ex, jl_module_t *module, jl_svec_t *sparam_vals) { jl_code_info_t *func = (jl_code_info_t*)jl_expand((jl_value_t*)ex, module); From 6a22b25b0b470aec65eb17c578bfde4bf922796a Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Thu, 3 Oct 2019 08:32:57 -0400 Subject: [PATCH 7/7] Add GC annotations inside `jl_expand_and_resolve` (I think I did this right? I couldn't get the static analysis to build on my machine...) --- src/method.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/method.c b/src/method.c index 41e31b8daeb97..c07c161e952a3 100644 --- a/src/method.c +++ b/src/method.c @@ -386,10 +386,12 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_method_t *def, jl_value_t *generator JL_DLLEXPORT jl_code_info_t *jl_expand_and_resolve(jl_value_t *ex, jl_module_t *module, jl_svec_t *sparam_vals) { jl_code_info_t *func = (jl_code_info_t*)jl_expand((jl_value_t*)ex, module); + JL_GC_PUSH1(&func); if (jl_is_code_info(func)) { jl_array_t *stmts = (jl_array_t*)func->code; jl_resolve_globals_in_ir(stmts, module, sparam_vals, 1); } + JL_GC_POP(); return func; }