task: drop parent and module references (#29483)

The parent link can unnecessarily hold memory references live, and the current-module function is gone.
Their uses have been deprecated in favor of the macros @`__FILE__` and @`__MODULE__`.
The `workspace()` function is also gone now, so we can remove the support code for it.

To implement detection of broken incremental compilation (and help
ensure the module is permanently rooted as is generally expected),
keep track of all "open" modules in a hashtable (jl_current_modules).
We can further refine this later, if desired (e.g. also allow any
submodule and of any item in `jl_module_init_order`).

Also removes the global `module_stack` state. This makes us a bit more
robust to async module definition. We could perhaps do even more here,
if necessary (e.g. by checking first that all parents are initialized,
instead of just the direct ancestor). But this is hopefully a vanishingly
rare situation anyways.

base/loading.jl

@@ -1013,22 +1013,16 @@ include_string(m::Module, txt::AbstractString, fname::AbstractString="string") =
     include_string(m, String(txt), String(fname))
 
 function source_path(default::Union{AbstractString,Nothing}="")
-    t = current_task()
-    while true
-        s = t.storage
-        if s !== nothing && haskey(s, :SOURCE_PATH)
-            return s[:SOURCE_PATH]
-        end
-        if t === t.parent
-            return default
-        end
-        t = t.parent
+    s = current_task().storage
+    if s !== nothing && haskey(s, :SOURCE_PATH)
+        return s[:SOURCE_PATH]
     end
+    return default
 end
 
 function source_dir()
     p = source_path(nothing)
-    p === nothing ? pwd() : dirname(p)
+    return p === nothing ? pwd() : dirname(p)
 end
 
 include_relative(mod::Module, path::AbstractString) = include_relative(mod, String(path))

src/builtins.c

@@ -596,74 +596,6 @@ JL_CALLABLE(jl_f__apply_latest)
     return ret;
 }
 
-// eval -----------------------------------------------------------------------
-
-JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex)
-{
-    jl_ptls_t ptls = jl_get_ptls_states();
-    if (ptls->in_pure_callback)
-        jl_error("eval cannot be used in a generated function");
-    jl_value_t *v = NULL;
-    int last_lineno = jl_lineno;
-    size_t last_age = ptls->world_age;
-    jl_module_t *last_m = ptls->current_module;
-    jl_module_t *task_last_m = ptls->current_task->current_module;
-    if (jl_options.incremental && jl_generating_output()) {
-        if (m != last_m) {
-            jl_printf(JL_STDERR, "WARNING: eval from module %s to %s:    \n",
-                      jl_symbol_name(m->name), jl_symbol_name(last_m->name));
-            jl_static_show(JL_STDERR, ex);
-            jl_printf(JL_STDERR, "\n  ** incremental compilation may be broken for this module **\n\n");
-        }
-    }
-    JL_TRY {
-        ptls->current_task->current_module = ptls->current_module = m;
-        ptls->world_age = jl_world_counter;
-        v = jl_toplevel_eval(m, ex);
-    }
-    JL_CATCH {
-        jl_lineno = last_lineno;
-        ptls->current_module = last_m;
-        ptls->current_task->current_module = task_last_m;
-        jl_rethrow();
-    }
-    jl_lineno = last_lineno;
-    ptls->world_age = last_age;
-    ptls->current_module = last_m;
-    ptls->current_task->current_module = task_last_m;
-    assert(v);
-    return v;
-}
-
-JL_CALLABLE(jl_f_isdefined)
-{
-    jl_module_t *m = NULL;
-    jl_sym_t *s = NULL;
-    JL_NARGS(isdefined, 2, 2);
-    if (!jl_is_module(args[0])) {
-        jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(args[0]);
-        assert(jl_is_datatype(vt));
-        size_t idx;
-        if (jl_is_long(args[1])) {
-            idx = jl_unbox_long(args[1])-1;
-            if (idx >= jl_datatype_nfields(vt))
-                return jl_false;
-        }
-        else {
-            JL_TYPECHK(isdefined, symbol, args[1]);
-            idx = jl_field_index(vt, (jl_sym_t*)args[1], 0);
-            if ((int)idx == -1)
-                return jl_false;
-        }
-        return jl_field_isdefined(args[0], idx) ? jl_true : jl_false;
-    }
-    JL_TYPECHK(isdefined, module, args[0]);
-    JL_TYPECHK(isdefined, symbol, args[1]);
-    m = (jl_module_t*)args[0];
-    s = (jl_sym_t*)args[1];
-    return jl_boundp(m, s) ? jl_true : jl_false;
-}
-
 // tuples ---------------------------------------------------------------------
 
 JL_CALLABLE(jl_f_tuple)
@@ -840,6 +772,36 @@ JL_CALLABLE(jl_f_nfields)
     return jl_box_long(jl_field_count(jl_typeof(x)));
 }
 
+JL_CALLABLE(jl_f_isdefined)
+{
+    jl_module_t *m = NULL;
+    jl_sym_t *s = NULL;
+    JL_NARGS(isdefined, 2, 2);
+    if (!jl_is_module(args[0])) {
+        jl_datatype_t *vt = (jl_datatype_t*)jl_typeof(args[0]);
+        assert(jl_is_datatype(vt));
+        size_t idx;
+        if (jl_is_long(args[1])) {
+            idx = jl_unbox_long(args[1]) - 1;
+            if (idx >= jl_datatype_nfields(vt))
+                return jl_false;
+        }
+        else {
+            JL_TYPECHK(isdefined, symbol, args[1]);
+            idx = jl_field_index(vt, (jl_sym_t*)args[1], 0);
+            if ((int)idx == -1)
+                return jl_false;
+        }
+        return jl_field_isdefined(args[0], idx) ? jl_true : jl_false;
+    }
+    JL_TYPECHK(isdefined, module, args[0]);
+    JL_TYPECHK(isdefined, symbol, args[1]);
+    m = (jl_module_t*)args[0];
+    s = (jl_sym_t*)args[1];
+    return jl_boundp(m, s) ? jl_true : jl_false;
+}
+
+
 // apply_type -----------------------------------------------------------------
 
 static int valid_type_param(jl_value_t *v)
@@ -1020,7 +982,6 @@ JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name, jl_value_t *lb, jl_value_
 
 JL_CALLABLE(jl_f__typevar)
 {
-    jl_ptls_t ptls = jl_get_ptls_states();
     JL_NARGS(TypeVar, 3, 3);
     JL_TYPECHK(arraysize, symbol, args[0]);
     return (jl_value_t *)jl_new_typevar((jl_sym_t*)args[0], args[1], args[2]);

src/gc.c

@@ -2439,10 +2439,6 @@ extern jl_array_t *jl_all_methods;
 static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp,
                                      jl_ptls_t ptls2)
 {
-    // `current_module` might not have a value when the thread is not
-    // running.
-    if (ptls2->current_module)
-        gc_mark_queue_obj(gc_cache, sp, ptls2->current_module);
     gc_mark_queue_obj(gc_cache, sp, ptls2->current_task);
     gc_mark_queue_obj(gc_cache, sp, ptls2->root_task);
     gc_mark_queue_obj(gc_cache, sp, ptls2->exception_in_transit);
@@ -2453,13 +2449,17 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp)
 {
     // modules
     gc_mark_queue_obj(gc_cache, sp, jl_main_module);
-    gc_mark_queue_obj(gc_cache, sp, jl_internal_main_module);
 
     // invisible builtin values
     if (jl_an_empty_vec_any != NULL)
         gc_mark_queue_obj(gc_cache, sp, jl_an_empty_vec_any);
     if (jl_module_init_order != NULL)
         gc_mark_queue_obj(gc_cache, sp, jl_module_init_order);
+    for (size_t i = 0; i < jl_current_modules.size; i += 2) {
+        if (jl_current_modules.table[i + 1] != HT_NOTFOUND) {
+            gc_mark_queue_obj(gc_cache, sp, jl_current_modules.table[i]);
+        }
+    }
     if (jl_cfunction_list != NULL)
         gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list);
     gc_mark_queue_obj(gc_cache, sp, jl_anytuple_type_type);

src/init.c

@@ -612,6 +612,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     jl_ptls_t ptls = jl_get_ptls_states();
     jl_safepoint_init();
     libsupport_init();
+    htable_new(&jl_current_modules, 0);
     ios_set_io_wait_func = jl_set_io_wait;
     jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.),
                                     // best to call this first, since it also initializes libuv
@@ -728,18 +729,10 @@ void _julia_init(JL_IMAGE_SEARCH rel)
         jl_core_module = jl_new_module(jl_symbol("Core"));
         jl_type_typename->mt->module = jl_core_module;
         jl_top_module = jl_core_module;
-        ptls->current_module = jl_core_module;
         jl_init_intrinsic_functions();
         jl_init_primitives();
         jl_get_builtins();
-
-        jl_new_main_module();
-        jl_internal_main_module = jl_main_module;
-
-        ptls->current_module = jl_core_module;
-        for (int t = 0; t < jl_n_threads; t++) {
-            jl_all_tls_states[t]->root_task->current_module = jl_core_module;
-        }
+        jl_init_main_module();
 
         jl_load(jl_core_module, "boot.jl");
         jl_get_builtin_hooks();
@@ -784,10 +777,6 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     if (jl_base_module != NULL) {
         jl_add_standard_imports(jl_main_module);
     }
-    ptls->current_module = jl_main_module;
-    for (int t = 0; t < jl_n_threads; t++) {
-        jl_all_tls_states[t]->root_task->current_module = jl_main_module;
-    }
 
     // This needs to be after jl_start_threads
     if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON)

src/interpreter.c

@@ -904,28 +904,13 @@ SECT_INTERP CALLBACK_ABI void *jl_interpret_toplevel_expr_in_callback(interprete
     struct interpret_toplevel_expr_in_args *args =
         (struct interpret_toplevel_expr_in_args*)vargs;
     JL_GC_PROMISE_ROOTED(args);
-    jl_ptls_t ptls = jl_get_ptls_states();
-    jl_value_t *v=NULL;
-    jl_module_t *last_m = ptls->current_module;
-    jl_module_t *task_last_m = ptls->current_task->current_module;
     s->src = args->src;
     s->module = args->m;
     s->sparam_vals = args->sparam_vals;
     s->preevaluation = (s->sparam_vals != NULL);
     s->continue_at = 0;
     s->mi = NULL;
-
-    JL_TRY {
-        ptls->current_task->current_module = ptls->current_module = args->m;
-        v = eval_value(args->e, s);
-    }
-    JL_CATCH {
-        ptls->current_module = last_m;
-        ptls->current_task->current_module = task_last_m;
-        jl_rethrow();
-    }
-    ptls->current_module = last_m;
-    ptls->current_task->current_module = task_last_m;
+    jl_value_t *v = eval_value(args->e, s);
     assert(v);
     return (void*)v;
 }

src/jlapi.c

@@ -86,14 +86,11 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str)
 {
     jl_value_t *r;
     JL_TRY {
-        const char *filename = "none";
+        const char filename[] = "none";
         jl_value_t *ast = jl_parse_input_line(str, strlen(str),
                 filename, strlen(filename));
         JL_GC_PUSH1(&ast);
-        size_t last_age = jl_get_ptls_states()->world_age;
-        jl_get_ptls_states()->world_age = jl_get_world_counter();
         r = jl_toplevel_eval_in(jl_main_module, ast);
-        jl_get_ptls_states()->world_age = last_age;
         JL_GC_POP();
         jl_exception_clear();
     }

src/julia.h

@@ -1338,7 +1338,6 @@ JL_DLLEXPORT const char *jl_string_ptr(jl_value_t *s);
 
 // modules and global variables
 extern JL_DLLEXPORT jl_module_t *jl_main_module JL_GLOBALLY_ROOTED;
-extern JL_DLLEXPORT jl_module_t *jl_internal_main_module JL_GLOBALLY_ROOTED;
 extern JL_DLLEXPORT jl_module_t *jl_core_module JL_GLOBALLY_ROOTED;
 extern JL_DLLEXPORT jl_module_t *jl_base_module JL_GLOBALLY_ROOTED;
 extern JL_DLLEXPORT jl_module_t *jl_top_module JL_GLOBALLY_ROOTED;
@@ -1372,7 +1371,6 @@ JL_DLLEXPORT void jl_module_import(jl_module_t *to, jl_module_t *from,
 JL_DLLEXPORT void jl_module_export(jl_module_t *from, jl_sym_t *s);
 JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s);
 JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var) JL_NOTSAFEPOINT;
-JL_DLLEXPORT jl_module_t *jl_new_main_module(void);
 JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m);
 STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name)
 {
@@ -1599,7 +1597,6 @@ typedef struct _jl_handler_t {
 
 typedef struct _jl_task_t {
     JL_DATA_TYPE
-    struct _jl_task_t *parent;
     jl_value_t *tls;
     jl_sym_t *state;
     jl_value_t *donenotify;
@@ -1620,8 +1617,6 @@ typedef struct _jl_task_t {
     jl_handler_t *eh;
     // saved gc stack top for context switches
     jl_gcframe_t *gcstack;
-    // current module, or NULL if this task has not set one
-    jl_module_t *current_module;
     // current world age
     size_t world_age;
 
@@ -1932,7 +1927,6 @@ typedef struct {
     float value;
 } jl_nullable_float32_t;
 
-#define jl_current_module (jl_get_ptls_states()->current_module)
 #define jl_current_task (jl_get_ptls_states()->current_task)
 #define jl_root_task (jl_get_ptls_states()->root_task)
 #define jl_exception_in_transit (jl_get_ptls_states()->exception_in_transit)

src/julia_internal.h

@@ -425,6 +425,7 @@ void jl_assign_bits(void *dest, jl_value_t *bits) JL_NOTSAFEPOINT;
 jl_expr_t *jl_exprn(jl_sym_t *head, size_t n);
 jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module);
 jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st, int iskw);
+void jl_init_main_module(void);
 int jl_is_submodule(jl_module_t *child, jl_module_t *parent);
 jl_array_t *jl_get_loaded_modules(void);
 
@@ -460,6 +461,7 @@ jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims,
 void jl_module_run_initializer(jl_module_t *m);
 jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var) JL_NOTSAFEPOINT;
 extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
+extern htable_t jl_current_modules JL_GLOBALLY_ROOTED;
 extern jl_array_t *jl_cfunction_list JL_GLOBALLY_ROOTED;
 
 #ifdef JL_USE_INTEL_JITEVENTS

src/julia_threads.h

@@ -150,7 +150,6 @@ struct _jl_tls_states_t {
     volatile int8_t in_finalizer;
     int8_t disable_gc;
     volatile sig_atomic_t defer_signal;
-    struct _jl_module_t *current_module;
     struct _jl_task_t *volatile current_task;
     struct _jl_task_t *root_task;
 //#ifdef COPY_STACKS

src/method.c

@@ -394,14 +394,10 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo)
     jl_ptls_t ptls = jl_get_ptls_states();
     int last_lineno = jl_lineno;
     int last_in = ptls->in_pure_callback;
-    jl_module_t *last_m = ptls->current_module;
-    jl_module_t *task_last_m = ptls->current_task->current_module;
     size_t last_age = jl_get_ptls_states()->world_age;
 
     JL_TRY {
         ptls->in_pure_callback = 1;
-        // need to eval macros in the right module
-        ptls->current_task->current_module = ptls->current_module = linfo->def.method->module;
         // and the right world
         ptls->world_age = def->min_world;
 
@@ -425,16 +421,12 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo)
 
         ptls->in_pure_callback = last_in;
         jl_lineno = last_lineno;
-        ptls->current_module = last_m;
-        ptls->current_task->current_module = task_last_m;
         ptls->world_age = last_age;
         jl_linenumber_to_lineinfo(func, def->module, def->name);
     }
     JL_CATCH {
         ptls->in_pure_callback = last_in;
         jl_lineno = last_lineno;
-        ptls->current_module = last_m;
-        ptls->current_task->current_module = task_last_m;
         jl_rethrow();
     }
     JL_GC_POP();

src/module.c

@@ -55,12 +55,15 @@ uint32_t jl_module_next_counter(jl_module_t *m)
 
 JL_DLLEXPORT jl_value_t *jl_f_new_module(jl_sym_t *name, uint8_t std_imports)
 {
+    // TODO: should we prohibit this during incremental compilation?
     jl_module_t *m = jl_new_module(name);
     JL_GC_PUSH1(&m);
-    m->parent = jl_main_module;
+    m->parent = jl_main_module; // TODO: this is a lie
     jl_gc_wb(m, m->parent);
-    if (std_imports) jl_add_standard_imports(m);
+    if (std_imports)
+        jl_add_standard_imports(m);
     JL_GC_POP();
+    // TODO: should we somehow try to gc-root this correctly?
     return (jl_value_t*)m;
 }