Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Pkg3-style package loading #24405

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -118,17 +118,19 @@ end

# these return either the array of modules loaded from the path / content given
# or an Exception that describes why it couldn't be loaded
function _include_from_serialized(content::Vector{UInt8})
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content))
function _include_from_serialized(from::Module, content::Vector{UInt8})
return ccall(:jl_restore_incremental_from_buf, Any,
(Any, Ptr{UInt8}, Int),
from, content, sizeof(content))
end
function _include_from_serialized(path::String)
return ccall(:jl_restore_incremental, Any, (Cstring,), path)
function _include_from_serialized(from::Module, path::String)
return ccall(:jl_restore_incremental, Any, (Any, Cstring), from, path)
end

# returns an array of modules loaded, or an Exception that describes why it failed
# and it reconnects the Base.Docs.META
function _require_from_serialized(mod::Symbol, path_to_try::String)
restored = _include_from_serialized(path_to_try)
function _require_from_serialized(from::Module, mod::Symbol, path_to_try::String)
restored = _include_from_serialized(from, path_to_try)
if !isa(restored, Exception)
for M in restored::Vector{Any}
if isdefined(M, Base.Docs.META)
Expand All @@ -142,13 +144,13 @@ end
# returns `true` if require found a precompile cache for this mod, but couldn't load it
# returns `false` if the module isn't known to be precompilable
# returns the set of modules restored if the cache load succeeded
function _require_search_from_serialized(mod::Symbol, sourcepath::String)
function _require_search_from_serialized(from::Module, mod::Symbol, sourcepath::String)
paths = find_all_in_cache_path(mod)
for path_to_try in paths::Vector{String}
if stale_cachefile(sourcepath, path_to_try)
continue
end
restored = _require_from_serialized(mod, path_to_try)
restored = _require_from_serialized(from, mod, path_to_try)
if isa(restored, Exception)
if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.")
# can't use this cache due to a module uuid mismatch,
Expand Down Expand Up @@ -262,22 +264,22 @@ end
Force reloading of a package, even if it has been loaded before. This is intended for use
during package development as code is modified.
"""
function reload(name::AbstractString)
function reload(from::Module, name::AbstractString)
if contains(name, Filesystem.path_separator) || contains(name, ".")
# for reload("path/file.jl") just ask for include instead
error("use `include` instead of `reload` to load source files")
else
# reload("Package") is ok
unreference_module(Symbol(name))
require(Symbol(name))
require(from, Symbol(name))
end
end

# require always works in Main scope and loads files from node 1
const toplevel_load = Ref(true)

"""
require(module::Symbol)
require(from::Module, module::Symbol)

This function is part of the implementation of `using` / `import`, if a module is not
already defined in `Main`. It can also be called directly to force reloading a module,
Expand All @@ -295,16 +297,17 @@ then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on
all platforms, including those with case-insensitive filesystems like macOS and
Windows.
"""
function require(mod::Symbol)
function require(from::Module, mod::Symbol)
if !root_module_exists(mod)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if this check is still sensible. It depends on what it means to require a root module from within an other module.
Currently we are using root-modules for the stdlib packages, since we don't want to make them available under Main immediately. As I noted on Slack this means that we can't update stdlib packages that are backed into the sysimage.

_require(mod)
info("@$(getpid()): require($from, $mod)")
_require(from, mod)
# After successfully loading, notify downstream consumers
if toplevel_load[] && myid() == 1 && nprocs() > 1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This codepath is moving out of base with #25139, so the callback below will need to also be changed to include from.

# broadcast top-level import/using from node 1 (only)
@sync for p in procs()
p == 1 && continue
@async remotecall_wait(p) do
require(mod)
require(from, mod)
nothing
end
end
Expand Down Expand Up @@ -371,7 +374,7 @@ function register_all(a)
end
end

function _require(mod::Symbol)
function _require(from::Module, mod::Symbol)
# dependency-tracking is only used for one top-level include(path),
# and is not applied recursively to imported modules:
old_track_dependencies = _track_dependencies[]
Expand Down Expand Up @@ -400,7 +403,7 @@ function _require(mod::Symbol)
# attempt to load the module file via the precompile cache locations
doneprecompile = false
if JLOptions().use_compiled_modules != 0
doneprecompile = _require_search_from_serialized(mod, path)
doneprecompile = _require_search_from_serialized(from, mod, path)
if !isa(doneprecompile, Bool)
register_all(doneprecompile)
return
Expand All @@ -424,7 +427,7 @@ function _require(mod::Symbol)
# spawn off a new incremental pre-compile task for recursive `require` calls
# or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
cachefile = compilecache(mod)
m = _require_from_serialized(mod, cachefile)
m = _require_from_serialized(from, mod, cachefile)
if isa(m, Exception)
warn("The call to compilecache failed to create a usable precompiled cache file for module $name. Got:")
warn(m, prefix="WARNING: ")
Expand All @@ -446,7 +449,7 @@ function _require(mod::Symbol)
end
# the file requested `__precompile__`, so try to build a cache file and use that
cachefile = compilecache(mod)
m = _require_from_serialized(mod, cachefile)
m = _require_from_serialized(from, mod, cachefile)
if isa(m, Exception)
warn(m, prefix="WARNING: ")
# TODO: disable __precompile__(true) error and do normal include instead of error
Expand Down
4 changes: 2 additions & 2 deletions base/sysimg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,8 @@ unshift!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
Base.init_load_path(ccall(:jl_get_julia_home, Any, ()))

# load some stdlib packages but don't put their names in Main
Base.require(:DelimitedFiles)
Base.require(:Test)
Base.require(Base, :DelimitedFiles)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I am in favour of not loading stdlib packages into the sys-image at all and instead create cache-files for them during the build.

If we keep loading them they shouldn't go into Base in my opinion since people might continue to use using Base.Test which is something I think we don't want.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since changing this is non-breaking, we do not need to figure out the perfect way to build and give people stdlib packages in 1.0. For now, building them into the sysimg is easiest.

Base.require(Base, :Test)

empty!(LOAD_PATH)

Expand Down
18 changes: 9 additions & 9 deletions src/dump.c
Original file line number Diff line number Diff line change
Expand Up @@ -1921,7 +1921,7 @@ static int size_isgreater(const void *a, const void *b)
return *(size_t*)b - *(size_t*)a;
}

static jl_value_t *read_verify_mod_list(ios_t *s, arraylist_t *dependent_worlds)
static jl_value_t *read_verify_mod_list(jl_module_t *from, ios_t *s, arraylist_t *dependent_worlds)
{
if (!jl_main_module->uuid) {
return jl_get_exceptionf(jl_errorexception_type,
Expand All @@ -1944,9 +1944,9 @@ static jl_value_t *read_verify_mod_list(ios_t *s, arraylist_t *dependent_worlds)
static jl_value_t *require_func = NULL;
if (!require_func)
require_func = jl_get_global(jl_base_module, jl_symbol("require"));
jl_value_t *reqargs[2] = {require_func, (jl_value_t*)sym};
jl_value_t *reqargs[3] = {require_func, (jl_value_t*)from, (jl_value_t*)sym};
JL_TRY {
m = (jl_module_t*)jl_apply(reqargs, 2);
m = (jl_module_t*)jl_apply(reqargs, 3);
}
JL_CATCH {
ios_close(s);
Expand Down Expand Up @@ -2655,7 +2655,7 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure)
return 1;
}

static jl_value_t *_jl_restore_incremental(ios_t *f)
static jl_value_t *_jl_restore_incremental(jl_module_t *from, ios_t *f)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (ios_eof(f) || !jl_read_verify_header(f)) {
Expand All @@ -2678,7 +2678,7 @@ static jl_value_t *_jl_restore_incremental(ios_t *f)
arraylist_new(&dependent_worlds, 0);

// verify that the system state is valid
jl_value_t *verify_result = read_verify_mod_list(f, &dependent_worlds);
jl_value_t *verify_result = read_verify_mod_list(from, f, &dependent_worlds);
if (!jl_is_array(verify_result)) {
arraylist_free(&dependent_worlds);
ios_close(f);
Expand Down Expand Up @@ -2747,21 +2747,21 @@ static jl_value_t *_jl_restore_incremental(ios_t *f)
return (jl_value_t*)restored;
}

JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz)
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(jl_module_t *from, const char *buf, size_t sz)
{
ios_t f;
ios_static_buffer(&f, (char*)buf, sz);
return _jl_restore_incremental(&f);
return _jl_restore_incremental(from, &f);
}

JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname)
JL_DLLEXPORT jl_value_t *jl_restore_incremental(jl_module_t *from, const char *fname)
{
ios_t f;
if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) {
return jl_get_exceptionf(jl_errorexception_type,
"Cache file \"%s\" not found.\n", fname);
}
return _jl_restore_incremental(&f);
return _jl_restore_incremental(from, &f);
}

// --- init ---
Expand Down
4 changes: 2 additions & 2 deletions src/julia.h
Original file line number Diff line number Diff line change
Expand Up @@ -1356,8 +1356,8 @@ JL_DLLEXPORT void jl_save_system_image(const char *fname);
JL_DLLEXPORT void jl_restore_system_image(const char *fname);
JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len);
JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist);
JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname);
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz);
JL_DLLEXPORT jl_value_t *jl_restore_incremental(jl_module_t *from, const char *fname);
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(jl_module_t *from, const char *buf, size_t sz);

// front end interface
JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
Expand Down
10 changes: 5 additions & 5 deletions src/toplevel.c
Original file line number Diff line number Diff line change
Expand Up @@ -363,15 +363,15 @@ static int jl_eval_expr_with_compiler_p(jl_value_t *e, int compileloops, jl_modu
return 0;
}

static jl_module_t *call_require(jl_sym_t *var)
static jl_module_t *call_require(jl_module_t *from, jl_sym_t *var)
{
static jl_value_t *require_func = NULL;
jl_module_t *m = NULL;
if (require_func == NULL && jl_base_module != NULL)
require_func = jl_get_global(jl_base_module, jl_symbol("require"));
if (require_func != NULL) {
jl_value_t *reqargs[2] = {require_func, (jl_value_t*)var};
m = (jl_module_t*)jl_apply(reqargs, 2);
jl_value_t *reqargs[3] = {require_func, (jl_value_t*)from, (jl_value_t*)var};
m = (jl_module_t*)jl_apply(reqargs, 3);
}
if (m == NULL || !jl_is_module(m)) {
jl_errorf("failed to load module %s", jl_symbol_name(var));
Expand Down Expand Up @@ -400,7 +400,7 @@ static jl_module_t *eval_import_path(jl_module_t *from, jl_array_t *args, jl_sym
m = jl_base_module;
}
else {
m = call_require(var);
m = call_require(from, var);
}
if (i == jl_array_len(args))
return m;
Expand Down Expand Up @@ -484,7 +484,7 @@ static jl_module_t *deprecation_replacement_module(jl_module_t *parent, jl_sym_t
{
if (parent == jl_base_module) {
if (name == jl_symbol("Test") || name == jl_symbol("Mmap"))
return call_require(name);
return call_require(parent, name);
}
return NULL;
}
Expand Down
20 changes: 10 additions & 10 deletions test/compile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -150,14 +150,14 @@ try
@test __precompile__(true) === nothing

# Issue #21307
Base.require(Foo2_module)
Base.require(Main, Foo2_module)
@eval let Foo2_module = $(QuoteNode(Foo2_module)), # use @eval to see the results of loading the compile
Foo = root_module(Foo2_module)
Foo.override(::Int) = 'a'
Foo.override(::Float32) = 'b'
end

Base.require(Foo_module)
Base.require(Main, Foo_module)

@eval let Foo_module = $(QuoteNode(Foo_module)), # use @eval to see the results of loading the compile
Foo = root_module(Foo_module)
Expand All @@ -175,7 +175,7 @@ try
# use _require_from_serialized to ensure that the test fails if
# the module doesn't reload from the image:
@test_warn "WARNING: replacing module $Foo_module." begin
ms = Base._require_from_serialized(Foo_module, cachefile)
ms = Base._require_from_serialized(Main, Foo_module, cachefile)
@test isa(ms, Array{Any,1})
Base.register_all(ms)
end
Expand Down Expand Up @@ -319,7 +319,7 @@ try
fb_uuid1 = Base.module_uuid(FooBar1)
@test fb_uuid != fb_uuid1

reload("FooBar")
reload(Main, "FooBar")
@test fb_uuid != Base.module_uuid(root_module(:FooBar))
@test fb_uuid1 == Base.module_uuid(FooBar1)
fb_uuid = Base.module_uuid(root_module(:FooBar))
Expand All @@ -328,7 +328,7 @@ try
@test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji"))
@test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji"))

reload("FooBar1")
reload(Main, "FooBar1")
@test fb_uuid == Base.module_uuid(root_module(:FooBar))
@test fb_uuid1 != Base.module_uuid(root_module(:FooBar1))

Expand All @@ -354,7 +354,7 @@ try
end
""")
@test_warn "ERROR: LoadError: break me\nStacktrace:\n [1] error" try
Base.require(:FooBar2)
Base.require(Main, :FooBar2)
error("\"LoadError: break me\" test failed")
catch exc
isa(exc, ErrorException) || rethrow(exc)
Expand Down Expand Up @@ -394,7 +394,7 @@ try
""")
rm(FooBarT_file)
@test Base.stale_cachefile(FooBarT2_file, joinpath(dir2, "FooBarT2.ji"))
@test Base.require(:FooBarT2) isa Module
@test Base.require(Main, :FooBarT2) isa Module
finally
splice!(Base.LOAD_CACHE_PATH, 1:2)
splice!(LOAD_PATH, 1)
Expand Down Expand Up @@ -527,7 +527,7 @@ let dir = mktempdir()
""")
Base.compilecache("$(Test2_module)")
@test !Base.isbindingresolved(Main, Test2_module)
Base.require(Test2_module)
Base.require(Main, Test2_module)
@test take!(loaded_modules) == Test1_module
@test take!(loaded_modules) == Test2_module
write(joinpath(dir, "$(Test3_module).jl"),
Expand All @@ -536,7 +536,7 @@ let dir = mktempdir()
using $(Test3_module)
end
""")
Base.require(Test3_module)
Base.require(Main, Test3_module)
@test take!(loaded_modules) == Test3_module
finally
pop!(Base.package_callbacks)
Expand All @@ -554,7 +554,7 @@ let module_name = string("a",randstring())
touch(file_name)
code = """module $(module_name)\nend\n"""
write(file_name, code)
reload(module_name)
reload(Main, module_name)
@test isa(root_module(Symbol(module_name)), Module)
@test shift!(LOAD_PATH) == path
rm(file_name)
Expand Down