WIP: static compile part 4 (user-interface)

NEWS.md

@@ -31,6 +31,12 @@ New language features
   * The syntax `function foo end` can be used to introduce a generic function without
     yet adding any methods ([#8283]).
 
+  * Incremental compilation of modules: ``Base.compile(module::Symbol)`` (stored in `~/.julia/lib/v0.4`)
+
+      * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.
+
+      * New option `--compile-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line.
+
 Language changes
 ----------------
 
@@ -450,6 +456,8 @@ Deprecated or removed
 
     * `sync_gc_total_bytes` -> `jl_gc_sync_total_bytes`
 
+  * `require(::AbstractString)` and `reload` (see news about addition of `compile`)
+
 Julia v0.3.0 Release Notes
 ==========================
 

base/boot.jl

@@ -71,8 +71,8 @@
 #    module::Module
 #end
 
-#type Box{T}
-#    contents::T
+#type Box
+#    contents::Any
 #end
 
 #abstract Ref{T}

base/client.jl

@@ -274,7 +274,9 @@ let reqarg = Set(UTF8String["--home",          "-H",
             end
             # load file immediately on all processors
             if opts.load != C_NULL
-                require(bytestring(opts.load))
+                @sync for p in procs()
+                    @async remotecall_fetch(p, include, bytestring(opts.load))
+                end
             end
             # eval expression
             if opts.eval != C_NULL
@@ -322,13 +324,16 @@ is_interactive = false
 isinteractive() = (is_interactive::Bool)
 
 const LOAD_PATH = ByteString[]
+const LOAD_CACHE_PATH = ByteString[]
 function init_load_path()
     vers = "v$(VERSION.major).$(VERSION.minor)"
     if haskey(ENV,"JULIA_LOAD_PATH")
         prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @windows? ';' : ':'))
     end
     push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers))
     push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers))
+    push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","lib",vers))
+    push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme
 end
 
 function load_juliarc()

base/deprecated.jl

@@ -635,7 +635,6 @@ end
 @deprecate mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Integer}, s::IOStream, offset::FileOffset=position(s)) mmap(s, BitArray, dims, offset)
 @deprecate mmap_bitarray{N}(dims::NTuple{N,Integer}, s::IOStream, offset=position(s)) mmap(s, BitArray, dims, offset)
 
-
 # T[a:b] and T[a:s:b]
 @noinline function getindex{T<:Union{Char,Number}}(::Type{T}, r::Range)
     depwarn("T[a:b] concatenation is deprecated; use T[a:b;] instead", :getindex)
@@ -654,3 +653,34 @@ function getindex{T<:Union{Char,Number}}(::Type{T}, r1::Range, rs::Range...)
     end
     return a
 end
+
+function require(mod::AbstractString)
+    depwarn("`require` is deprecated, use `using` or `import` instead", :require)
+    require(symbol(require_filename(mod)))
+end
+function require(f::AbstractString, fs::AbstractString...)
+    require(f)
+    for fn in fs
+        require(fn)
+    end
+end
+export require
+function require_filename(name::AbstractString)
+    # This function can be deleted when the deprecation for `require`
+    # is deleted.
+    # While we could also strip off the absolute path, the user may be
+    # deliberately directing to a different file than what got
+    # cached. So this takes a conservative approach.
+    if endswith(name, ".jl")
+        tmp = name[1:end-3]
+        for prefix in LOAD_CACHE_PATH
+            path = joinpath(prefix, tmp*".ji")
+            if isfile(path)
+                return tmp
+            end
+        end
+    end
+    name
+end
+const reload = require
+export reload

base/exports.jl

@@ -1096,8 +1096,6 @@ export
     evalfile,
     include,
     include_string,
-    reload,
-    require,
 
 # RTS internals
     finalizer,

base/interactiveutil.jl

@@ -385,7 +385,6 @@ function workspace()
          Expr(:toplevel,
               :(const Base = $(Expr(:quote, b))),
               :(const LastMain = $(Expr(:quote, last)))))
-    empty!(package_list)
     empty!(package_locks)
     nothing
 end

base/loading.jl

@@ -1,6 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-# require
+# Base.require is the implementation for the `import` statement
 
 function find_in_path(name::AbstractString)
     isabspath(name) && return name
@@ -23,8 +23,8 @@ function find_in_path(name::AbstractString)
     return nothing
 end
 
-find_in_node1_path(name) = myid()==1 ?
-    find_in_path(name) : remotecall_fetch(1, find_in_path, name)
+find_in_node_path(name, node::Int=1) = myid() == node ?
+    find_in_path(name) : remotecall_fetch(node, find_in_path, name)
 
 function find_source_file(file)
     (isabspath(file) || isfile(file)) && return file
@@ -34,62 +34,114 @@ function find_source_file(file)
     isfile(file2) ? file2 : nothing
 end
 
-# Store list of files and their load time
-package_list = Dict{ByteString,Float64}()
-# to synchronize multiple tasks trying to require something
-package_locks = Dict{ByteString,Any}()
-require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs require(x); end)
-
-# only broadcast top-level (not nested) requires and reloads
-toplevel_load = true
+function find_in_cache_path(mod::Symbol)
+    name = string(mod)
+    for prefix in LOAD_CACHE_PATH
+        path = joinpath(prefix, name*".ji")
+        if isfile(path)
+            produce(path)
+        end
+    end
+    nothing
+end
 
-function require(name::AbstractString)
-    path = find_in_node1_path(name)
-    path == nothing && throw(ArgumentError("$name not found in path"))
+function _include_from_serialized(content::Vector{UInt8})
+    m = ccall(:jl_restore_incremental_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content))
+    return m != 0
+end
 
-    if myid() == 1 && toplevel_load
-        refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ]
-        _require(path)
-        for r in refs; wait(r); end
+function _require_from_serialized(node::Int, path_to_try::ByteString, toplevel_load::Bool)
+    if toplevel_load && myid() == 1 && nprocs() > 1
+        # broadcast top-level import/using from node 1 (only)
+        if node == myid()
+            content = open(readbytes, path_to_try)
+        else
+            content = remotecall_fetch(node, open, readbytes, path_to_try)
+        end
+        if _include_from_serialized(content)
+            others = filter(x -> x != myid(), procs())
+            refs = Any[ @spawnat p _include_from_serialized(content) for p in others]
+            for (id, ref) in zip(others, refs)
+                if !fetch(ref)
+                    warn("node state is inconsistent: node $id failed to load cache from $path_to_try")
+                end
+            end
+            return true
+        end
+    elseif node == myid()
+        if ccall(:jl_restore_incremental, UInt, (Ptr{Uint8},), path_to_try) != 0
+            return true
+        end
     else
-        _require(path)
+        content = remotecall_fetch(node, open, readbytes, path_to_try)
+        if _include_from_serialized(content)
+            return true
+        end
     end
-    nothing
+    # otherwise, continue search
+    return false
 end
 
-function _require(path)
-    global toplevel_load
-    if haskey(package_list,path)
-        loaded, c = package_locks[path]
-        !loaded && wait(c)
-    else
-        last = toplevel_load
-        toplevel_load = false
-        try
-            reload_path(path)
-        finally
-            toplevel_load = last
+function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
+    name = string(mod)
+    finder = @spawnat node @task find_in_cache_path(mod) # TODO: switch this to an explicit Channel
+    while true
+        path_to_try = remotecall_fetch(node, finder->consume(fetch(finder)), finder)
+        path_to_try === nothing && return false
+        if _require_from_serialized(node, path_to_try, toplevel_load)
+            return true
+        else
+            warn("deserialization checks failed while attempting to load cache from $path_to_try")
         end
     end
 end
 
-function reload(name::AbstractString)
+# to synchronize multiple tasks trying to import/using something
+package_locks = Dict{Symbol,Condition}()
+package_loaded = Set{Symbol}()
+
+# require always works in Main scope and loads files from node 1
+toplevel_load = true
+function require(mod::Symbol)
     global toplevel_load
-    path = find_in_node1_path(name)
-    path == nothing && throw(ArgumentError("$name not found in path"))
-    refs = nothing
-    if myid() == 1 && toplevel_load
-        refs = Any[ @spawnat p reload_path(path) for p in filter(x->x!=1, procs()) ]
+    loading = get(package_locks, mod, false)
+    if loading !== false
+        # load already in progress for this module
+        wait(loading)
+        return
     end
+    package_locks[mod] = Condition()
+
     last = toplevel_load
-    toplevel_load = false
     try
-        reload_path(path)
+        toplevel_load = false
+        if _require_from_serialized(1, mod, last)
+            return true
+        end
+        if JLOptions().incremental != 0
+            # spawn off a new incremental compile task from node 1 for recursive `require` calls
+            cachefile = compile(mod)
+            if !_require_from_serialized(1, cachefile, last)
+                warn("require failed to create a precompiled cache file")
+            end
+            return
+        end
+
+        name = string(mod)
+        path = find_in_node_path(name, 1)
+        path === nothing && throw(ArgumentError("$name not found in path"))
+        if last && myid() == 1 && nprocs() > 1
+            # broadcast top-level import/using from node 1 (only)
+            content = open(readall, path)
+            refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ]
+            for r in refs; wait(r); end
+        else
+            eval(Main, :(Base.include_from_node1($path)))
+        end
     finally
         toplevel_load = last
-    end
-    if refs !== nothing
-        for r in refs; wait(r); end
+        loading = pop!(package_locks, mod)
+        notify(loading, all=true)
     end
     nothing
 end
@@ -145,38 +197,62 @@ function include_from_node1(path::AbstractString)
     result
 end
 
-function reload_path(path::AbstractString)
-    had = haskey(package_list, path)
-    if !had
-        package_locks[path] = (false, Condition())
-    end
-    package_list[path] = time()
-    tls = task_local_storage()
-    prev = pop!(tls, :SOURCE_PATH, nothing)
-    try
-        eval(Main, :(Base.include_from_node1($path)))
-    catch e
-        had || delete!(package_list, path)
-        rethrow(e)
-    finally
-        if prev != nothing
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    reloaded, c = package_locks[path]
-    if !reloaded
-        package_locks[path] = (true, c)
-        notify(c, all=true)
-    end
-    nothing
-end
-
 function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[])
     return eval(Module(:__anon__),
                 Expr(:toplevel,
                      :(const ARGS = $args),
-                     :(eval(x) = Core.eval(__anon__,x)),
-                     :(eval(m,x) = Core.eval(m,x)),
-                     :(include($path))))
+                     :(eval(x) = Main.Core.eval(__anon__,x)),
+                     :(eval(m,x) = Main.Core.eval(m,x)),
+                     :(Main.Base.include($path))))
 end
 evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...])
+
+function create_expr_cache(input::AbstractString, output::AbstractString)
+    code_object = """
+        while !eof(STDIN)
+            eval(Main, deserialize(STDIN))
+        end
+        """
+    io, pobj = open(detach(setenv(`$(julia_cmd())
+            --output-ji $output --output-incremental=yes
+            --startup-file=no --history-file=no
+            --eval $code_object`,
+        ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w", STDOUT)
+    serialize(io, quote
+        empty!(Base.LOAD_PATH)
+        append!(Base.LOAD_PATH, $LOAD_PATH)
+        empty!(Base.LOAD_CACHE_PATH)
+        append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH)
+        empty!(Base.DL_LOAD_PATH)
+        append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH)
+    end)
+    source = source_path(nothing)
+    if source !== nothing
+        serialize(io, quote
+            task_local_storage()[:SOURCE_PATH] = $(source)
+        end)
+    end
+    serialize(io, :(Base.include($(abspath(input)))))
+    if source !== nothing
+        serialize(io, quote
+            delete!(task_local_storage(), :SOURCE_PATH)
+        end)
+    end
+    close(io)
+    wait(pobj)
+    return pobj
+end
+
+function compile(mod::Symbol)
+    myid() == 1 || error("can only compile from node 1")
+    name = string(mod)
+    path = find_in_path(name)
+    path === nothing && throw(ArgumentError("$name not found in path"))
+    cachepath = LOAD_CACHE_PATH[1]
+    if !isdir(cachepath)
+        mkpath(cachepath)
+    end
+    cachefile = abspath(cachepath, name*".ji")
+    create_expr_cache(path, cachefile)
+    return cachefile
+end

base/options.jl

@@ -32,6 +32,7 @@ immutable JLOptions
     outputbc::Ptr{UInt8}
     outputo::Ptr{UInt8}
     outputji::Ptr{UInt8}
+    incremental::Int8
 end
 
 JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions))