Add --pkgimages=existing. (#52573)

Equivalent of #50586; implements #51474. With `--pkgimages=existing`, it's possible to disable the (often slow) generation of package images, without losing the ability to use existing ones. That's important now that we're moving more and more packages outside of the system image, e.g., running with `--pkgimages=no` otherwise takes close to 30s here before the Pkg REPL is usable. The main motivation for this is PkgEval, where generating package images is not very useful, yet disabling generation of them makes each job (which requires Pkg to drive the test process) take a significantly longer time. For example, `--pkgimages=yes` vs `no`: ```julia ❯ JULIA_DEBUG=loading ./julia --project=Example.jl --pkgimages=yes # no precompilation of REPL.jl ┌ Debug: Loading object cache file /Users/tim/Julia/src/julia/build/dev/usr/share/julia/compiled/v1.11/REPL/u0gqU_XmENM.dylib for REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] └ @ Base loading.jl:1116 julia> using Example # short time precompiling + pkgimg generation for Example.jl ┌ Debug: Loading object cache file /Users/tim/.julia/compiled/v1.11/Example/lLvWP_tJaso.dylib for Example [7876af07-990d-54b4-ab0e-23690620f79a] └ @ Base loading.jl:1116 ``` ```julia ❯ JULIA_DEBUG=loading ./julia --project=Example.jl --pkgimages=no ┌ Debug: Rejecting cache file /Users/tim/Julia/src/julia/build/dev/usr/share/julia/compiled/v1.11/REPL/u0gqU_XmENM.ji for REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] since the flags are mismatched │ current session: use_pkgimages = false, debug_level = 1, check_bounds = 0, inline = true, opt_level = 2 │ cache file: use_pkgimages = true, debug_level = 1, check_bounds = 0, inline = true, opt_level = 2 └ @ Base loading.jl:3289 # long time precompiling REPL.jl ┌ Debug: Loading cache file /Users/tim/.julia/compiled/v1.11/REPL/u0gqU_CWvWI.ji for REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] └ @ Base loading.jl:1119 julia> using Example # short time precompiling Example ┌ Debug: Loading cache file /Users/tim/.julia/compiled/v1.11/Example/lLvWP_CWvWI.ji for Example [7876af07-990d-54b4-ab0e-23690620f79a] └ @ Base loading.jl:1119 ``` With the new `--pkgimages=existing`: ```julia ❯ JULIA_DEBUG=loading ./julia --project=Example.jl --pkgimages=existing # no precompilation of REPL.jl ┌ Debug: Loading object cache file /Users/tim/Julia/src/julia/build/dev/usr/share/julia/compiled/v1.11/REPL/u0gqU_XmENM.dylib for REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] └ @ Base loading.jl:1116 julia> using Example # short time precompiling Example ┌ Debug: Loading cache file /Users/tim/.julia/compiled/v1.11/Example/lLvWP_CWvWI.ji for Example [7876af07-990d-54b4-ab0e-23690620f79a] └ @ Base loading.jl:1119 ```
JuliaLang · Dec 19, 2023 · 91d87c6 · 91d87c6
1 parent 4f25e87
commit 91d87c6
Show file tree

Hide file tree

Showing 9 changed files with 145 additions and 20 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -50,6 +50,9 @@ Command-line option changes
 This is intended to unify script and compilation workflows, where code loading may happen
 in the compiler and execution of `Main.main` may happen in the resulting executable. For interactive use, there is no semantic
 difference between defining a `main` function and executing the code directly at the end of the script ([50974]).
+* The `--compiled-modules` and `--pkgimages` flags can now be set to `existing`, which will
+  cause Julia to consider loading existing cache files, but not to create new ones ([#50586]
+  and [#52573]).
 
 Multi-threading changes
 -----------------------

diff --git a/base/loading.jl b/base/loading.jl
@@ -1043,9 +1043,35 @@ function find_all_in_cache_path(pkg::PkgId)
         end
     end
     if length(paths) > 1
-        # allocating the sort vector is less expensive than using sort!(.. by=mtime), which would
-        # call the relatively slow mtime multiple times per path
-        p = sortperm(mtime.(paths), rev = true)
+        function sort_by(path)
+            # when using pkgimages, consider those cache files first
+            pkgimage = if JLOptions().use_pkgimages != 0
+                io = open(path, "r")
+                try
+                    if iszero(isvalid_cache_header(io))
+                        false
+                    else
+                        _, _, _, _, _, _, _, flags = parse_cache_header(io, path)
+                        CacheFlags(flags).use_pkgimages
+                    end
+                finally
+                    close(io)
+                end
+            else
+                false
+            end
+            (; pkgimage, mtime=mtime(path))
+        end
+        function sort_lt(a, b)
+            if a.pkgimage != b.pkgimage
+                return a.pkgimage < b.pkgimage
+            end
+            return a.mtime < b.mtime
+        end
+
+        # allocating the sort vector is less expensive than using sort!(.. by=sort_by),
+        # which would call the relatively slow mtime multiple times per path
+        p = sortperm(sort_by.(paths), lt=sort_lt, rev=true)
         return paths[p]
     else
         return paths
@@ -2431,10 +2457,12 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::
     end
 
     if output_o !== nothing
+        @debug "Generating object cache file for $pkg"
         cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing)
         opt_level = Base.JLOptions().opt_level
         opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes`
     else
+        @debug "Generating cache file for $pkg"
         cpu_target = nothing
         opts = `-O0 --output-ji $(output) --output-incremental=yes`
     end
@@ -2531,7 +2559,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
     # create a temporary file in `cachepath` directory, write the cache in it,
     # write the checksum, _and then_ atomically move the file to `cachefile`.
     mkpath(cachepath)
-    cache_objects = JLOptions().use_pkgimages != 0
+    cache_objects = JLOptions().use_pkgimages == 1
     tmppath, tmpio = mktemp(cachepath)
 
     if cache_objects

diff --git a/base/util.jl b/base/util.jl
@@ -207,6 +207,8 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio
     opts.can_inline == 0 && push!(addflags, "--inline=no")
     opts.use_compiled_modules == 0 && push!(addflags, "--compiled-modules=no")
     opts.use_compiled_modules == 2 && push!(addflags, "--compiled-modules=existing")
+    opts.use_pkgimages == 0 && push!(addflags, "--pkgimages=no")
+    opts.use_pkgimages == 2 && push!(addflags, "--pkgimages=existing")
     opts.opt_level == 2 || push!(addflags, "-O$(opts.opt_level)")
     opts.opt_level_min == 0 || push!(addflags, "--min-optlevel=$(opts.opt_level_min)")
     push!(addflags, "-g$(opts.debug_level)")
@@ -242,9 +244,6 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio
     if opts.use_sysimage_native_code == 0
         push!(addflags, "--sysimage-native-code=no")
     end
-    if opts.use_pkgimages == 0
-        push!(addflags, "--pkgimages=no")
-    end
     return `$julia -C $cpu_target -J$image_file $addflags`
 end
 

diff --git a/doc/man/julia.1 b/doc/man/julia.1
@@ -93,8 +93,12 @@ Enable or disable Julia's default signal handlers
 Use native code from system image if available
 
 .TP
---compiled-modules={yes*|no}
-Enable or disable incremental precompilation of modules
+--compiled-modules={yes*|no|existing}
+Enable or disable incremental precompilation of modules.
+
+.TP
+--pkgimages={yes*|no|existing}
+Enable or disable usage of native code caching in the form of pkgimages
 
 .TP
 -e, --eval <expr>

diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md
@@ -599,15 +599,19 @@ A few other points to be aware of:
    an error to do this, but you simply need to be prepared that the system will try to copy some
    of these and to create a single unique instance of others.
 
-It is sometimes helpful during module development to turn off incremental precompilation. The
-command line flag `--compiled-modules={yes|no}` enables you to toggle module precompilation on and
-off. When Julia is started with `--compiled-modules=no` the serialized modules in the compile cache
-are ignored when loading modules and module dependencies.
-More fine-grained control is available with `--pkgimages=no`, which suppresses only
-native-code storage during precompilation. `Base.compilecache` can still be called
-manually. The state of this command line flag is passed to `Pkg.build` to disable automatic
-precompilation triggering when installing, updating, and explicitly building packages.
+It is sometimes helpful during module development to turn off incremental precompilation.
+The command line flag `--compiled-modules={yes|no|existing}` enables you to toggle module
+precompilation on and off. When Julia is started with `--compiled-modules=no` the serialized
+modules in the compile cache are ignored when loading modules and module dependencies. In
+some cases, you may want to load existing precompiled modules, but not create new ones. This
+can be done by starting Julia with `--compiled-modules=existing`. More fine-grained control
+is available with `--pkgimages={yes|no|existing}`, which only affects native-code storage
+during precompilation. `Base.compilecache` can still be called manually. The state of this
+command line flag is passed to `Pkg.build` to disable automatic precompilation triggering
+when installing, updating, and explicitly building packages.
 
 You can also debug some precompilation failures with environment variables. Setting
-`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of compiled
-native code. See the **Developer Documentation** part of the Julia manual, where you will find further details in the section documenting Julia's internals under "Package Images".
+`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of
+compiled native code. See the **Developer Documentation** part of the Julia manual, where
+you will find further details in the section documenting Julia's internals under "Package
+Images".
diff --git a/src/jloptions.c b/src/jloptions.c
@@ -112,7 +112,7 @@ static const char opts[]  =
     "                            Use native code from system image if available\n"
     " --compiled-modules={yes*|no|existing}\n"
     "                            Enable or disable incremental precompilation of modules\n"
-    " --pkgimages={yes*|no}\n"
+    " --pkgimages={yes*|no|existing}\n"
     "                            Enable or disable usage of native code caching in the form of pkgimages ($)\n\n"
 
     // actions
@@ -472,6 +472,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
                 jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_YES;
             else if (!strcmp(optarg,"no"))
                 jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_NO;
+            else if (!strcmp(optarg,"existing"))
+                jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_EXISTING;
             else
                 jl_errorf("julia: invalid argument to --pkgimages={yes|no} (%s)", optarg);
             break;

diff --git a/src/julia.h b/src/julia.h
@@ -2484,6 +2484,7 @@ JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT;
 #define JL_OPTIONS_USE_COMPILED_MODULES_YES 1
 #define JL_OPTIONS_USE_COMPILED_MODULES_NO 0
 
+#define JL_OPTIONS_USE_PKGIMAGES_EXISTING 2
 #define JL_OPTIONS_USE_PKGIMAGES_YES 1
 #define JL_OPTIONS_USE_PKGIMAGES_NO 0
 

diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c
@@ -619,6 +619,11 @@ JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags)
         return 1;
     }
 
+    // If package images are optional, ignore that bit (it will be unset in current_flags)
+    if (jl_options.use_pkgimages == JL_OPTIONS_USE_PKGIMAGES_EXISTING) {
+        flags &= ~1;
+    }
+
     // 2. Check all flags, execept opt level must be exact
     uint8_t mask = (1 << OPT_LEVEL)-1;
     if ((flags & mask) != (current_flags & mask))

diff --git a/test/loading.jl b/test/loading.jl
@@ -1406,3 +1406,82 @@ end
         end
     end
 end
+
+@testset "command-line flags" begin
+    mktempdir() do dir
+        # generate a Parent.jl and Child.jl package, with Parent depending on Child
+        open(joinpath(dir, "Child.jl"), "w") do io
+            println(io, """
+                module Child
+                end""")
+        end
+        open(joinpath(dir, "Parent.jl"), "w") do io
+            println(io, """
+                module Parent
+                using Child
+                end""")
+        end
+
+        # helper function to load a package and return the output
+        function load_package(name, args=``)
+            code = "using $name"
+            cmd = addenv(`$(Base.julia_cmd()) -e $code $args`,
+                        "JULIA_LOAD_PATH" => dir,
+                        "JULIA_DEBUG" => "loading")
+
+            out = Pipe()
+            proc = run(pipeline(cmd, stdout=out, stderr=out))
+            close(out.in)
+
+            log = @async String(read(out))
+            @test success(proc)
+            fetch(log)
+        end
+
+        log = load_package("Parent", `--compiled-modules=no --pkgimages=no`)
+        @test !occursin(r"Generating (cache|object cache) file", log)
+        @test !occursin(r"Loading (cache|object cache) file", log)
+
+
+        ## tests for `--compiled-modules`, which generates cache files
+
+        log = load_package("Child", `--compiled-modules=yes --pkgimages=no`)
+        @test occursin(r"Generating cache file for Child", log)
+        @test occursin(r"Loading cache file .+ for Child", log)
+
+        # with `--compiled-modules=existing` we should only precompile Child
+        log = load_package("Parent", `--compiled-modules=existing --pkgimages=no`)
+        @test !occursin(r"Generating cache file for Child", log)
+        @test occursin(r"Loading cache file .+ for Child", log)
+        @test !occursin(r"Generating cache file for Parent", log)
+        @test !occursin(r"Loading cache file .+ for Parent", log)
+
+        # the default is `--compiled-modules=yes`, which should now precompile Parent
+        log = load_package("Parent", `--pkgimages=no`)
+        @test !occursin(r"Generating cache file for Child", log)
+        @test occursin(r"Loading cache file .+ for Child", log)
+        @test occursin(r"Generating cache file for Parent", log)
+        @test occursin(r"Loading cache file .+ for Parent", log)
+
+
+        ## tests for `--pkgimages`, which generates object cache files
+
+        log = load_package("Child", `--compiled-modules=yes --pkgimages=yes`)
+        @test occursin(r"Generating object cache file for Child", log)
+        @test occursin(r"Loading object cache file .+ for Child", log)
+
+        # with `--pkgimages=existing` we should only generate code for Child
+        log = load_package("Parent", `--compiled-modules=yes --pkgimages=existing`)
+        @test !occursin(r"Generating object cache file for Child", log)
+        @test occursin(r"Loading object cache file .+ for Child", log)
+        @test !occursin(r"Generating object cache file for Parent", log)
+        @test !occursin(r"Loading object cache file .+ for Parent", log)
+
+        # the default is `--pkgimages=yes`, which should now generate code for Parent
+        log = load_package("Parent")
+        @test !occursin(r"Generating object cache file for Child", log)
+        @test occursin(r"Loading object cache file .+ for Child", log)
+        @test occursin(r"Generating object cache file for Parent", log)
+        @test occursin(r"Loading object cache file .+ for Parent", log)
+    end
+end