wip : move Distributed to stdlib

JuliaLang · Nov 22, 2017 · f94345e · f94345e
1 parent ea34968
commit f94345e
Show file tree

Hide file tree

Showing 35 changed files with 727 additions and 666 deletions.
diff --git a/base/client.jl b/base/client.jl
@@ -287,24 +287,10 @@ function process_options(opts::JLOptions)
     # remove filename from ARGS
     global PROGRAM_FILE = arg_is_program ? shift!(ARGS) : ""
 
-    # startup worker.
-    # opts.startupfile, opts.load, etc should should not be processed for workers.
-    if opts.worker == 1
-        # does not return
-        if opts.cookie != C_NULL
-            start_worker(unsafe_string(opts.cookie))
-        else
-            start_worker()
-        end
-    end
-
-    # add processors
-    if opts.nprocs > 0
-        addprocs(opts.nprocs)
-    end
-    # load processes from machine file
-    if opts.machinefile != C_NULL
-        addprocs(load_machine_file(unsafe_string(opts.machinefile)))
+    # Load Distributed module only if any of the Distributed options have been specified.
+    if (opts.worker == 1) || (opts.nprocs > 0) || (opts.machinefile != C_NULL)
+        eval(Main, :(using Distributed))
+        invokelatest(Main.Distributed.process_opts, opts)
     end
 
     # load ~/.juliarc file
@@ -319,8 +305,12 @@ function process_options(opts::JLOptions)
             println()
         elseif cmd == 'L'
             # load file immediately on all processors
-            @sync for p in procs()
-                @async remotecall_wait(include, p, Main, arg)
+            if nprocs() == 1
+                include(Main, arg)
+            else
+                @sync for p in invokelatest(Main.procs)
+                    @async invokelatest(Main.remotecall_wait, include, p, Main, arg)
+                end
             end
         end
     end
@@ -349,21 +339,6 @@ function load_juliarc()
     nothing
 end
 
-function load_machine_file(path::AbstractString)
-    machines = []
-    for line in split(read(path, String),'\n'; keep=false)
-        s = split(line, '*'; keep = false)
-        map!(strip, s, s)
-        if length(s) > 1
-            cnt = isnumber(s[1]) ? parse(Int,s[1]) : Symbol(s[1])
-            push!(machines,(s[2], cnt))
-        else
-            push!(machines,line)
-        end
-    end
-    return machines
-end
-
 import .Terminals
 import .REPL
 

diff --git a/base/deprecated.jl b/base/deprecated.jl
@@ -1355,6 +1355,42 @@ export conv, conv2, deconv, filt, filt!, xcorr
 @deprecate_moved PollingFileWatcher "FileWatching" true true
 @deprecate_moved watch_file "FileWatching" true true
 @deprecate_moved FileMonitor "FileWatching" true true
+# FIXME : This causes problems with the `myid` in loading.jl
+#@deprecate_binding Distributed nothing true ", run `using Distributed` instead"
+
+@eval @deprecate_moved $(Symbol("@spawn")) "Distributed" true true
+@eval @deprecate_moved $(Symbol("@spawnat")) "Distributed" true true
+@eval @deprecate_moved $(Symbol("@fetch")) "Distributed" true true
+@eval @deprecate_moved $(Symbol("@fetchfrom")) "Distributed" true true
+@eval @deprecate_moved $(Symbol("@everywhere")) "Distributed" true true
+@eval @deprecate_moved $(Symbol("@parallel")) "Distributed" true true
+
+@deprecate_moved addprocs "Distributed" true true
+@deprecate_moved CachingPool "Distributed" true true
+@deprecate_moved clear! "Distributed" true true
+@deprecate_moved ClusterManager "Distributed" true true
+@deprecate_moved default_worker_pool "Distributed" true true
+@deprecate_moved init_worker "Distributed" true true
+@deprecate_moved interrupt "Distributed" true true
+@deprecate_moved launch "Distributed" true true
+@deprecate_moved manage "Distributed" true true
+@deprecate_moved nworkers "Distributed" true true
+@deprecate_moved pmap "Distributed" true true
+@deprecate_moved procs "Distributed" true true
+@deprecate_moved remote "Distributed" true true
+@deprecate_moved remotecall "Distributed" true true
+@deprecate_moved remotecall_fetch "Distributed" true true
+@deprecate_moved remotecall_wait "Distributed" true true
+@deprecate_moved remote_do "Distributed" true true
+@deprecate_moved rmprocs "Distributed" true true
+@deprecate_moved workers "Distributed" true true
+@deprecate_moved WorkerPool "Distributed" true true
+@deprecate_moved RemoteChannel "Distributed" true true
+@deprecate_moved Future "Distributed" true true
+@deprecate_moved WorkerConfig "Distributed" true true
+@deprecate_moved RemoteException "Distributed" true true
+@deprecate_moved ProcessExitedException "Distributed" true true
+
 
 @deprecate_moved crc32c "CRC32c" true true
 

diff --git a/base/exports.jl b/base/exports.jl
@@ -18,7 +18,6 @@ export
     Markdown,
     Threads,
     Iterators,
-    Distributed,
 
 # Types
     AbstractChannel,
@@ -1262,38 +1261,7 @@ export
     nzrange,
     nnz,
 
-# Distributed module re-exports
-    @spawn,
-    @spawnat,
-    @fetch,
-    @fetchfrom,
-    @everywhere,
-    @parallel,
-
-    addprocs,
-    CachingPool,
-    clear!,
-    ClusterManager,
-    default_worker_pool,
-    init_worker,
-    interrupt,
-    launch,
-    manage,
+# Minimal set of Distributed exports - useful for a program to check if running
+# in distributed mode or not.
     myid,
-    nprocs,
-    nworkers,
-    pmap,
-    procs,
-    remote,
-    remotecall,
-    remotecall_fetch,
-    remotecall_wait,
-    remote_do,
-    rmprocs,
-    workers,
-    WorkerPool,
-    RemoteChannel,
-    Future,
-    WorkerConfig,
-    RemoteException,
-    ProcessExitedException
+    nprocs
diff --git a/base/loading.jl b/base/loading.jl
@@ -295,18 +295,19 @@ then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on
 all platforms, including those with case-insensitive filesystems like macOS and
 Windows.
 """
+
+myid() = isdefined(Main, :Distributed) ? invokelatest(Main.Distributed.myid) : 1
+nprocs() = isdefined(Main, :Distributed) ? invokelatest(Main.Distributed.nprocs) : 1
+
 function require(mod::Symbol)
     if !root_module_exists(mod)
         _require(mod)
         # After successfully loading, notify downstream consumers
         if toplevel_load[] && myid() == 1 && nprocs() > 1
             # broadcast top-level import/using from node 1 (only)
-            @sync for p in procs()
+            @sync for p in invokelatest(Main.procs)
                 p == 1 && continue
-                @async remotecall_wait(p) do
-                    require(mod)
-                    nothing
-                end
+                @async invokelatest(Main.remotecall_wait, ()->(require(mod); nothing), p)
             end
         end
         for callback in package_callbacks