Merge pull request #28091 from JuliaLang/sk/cpu_threads

Rename Sys.CPU_CORES => Sys.CPU_THREADS

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
       - image: circleci/python:2.7
         environment:
           ARCH: i686
-          JULIA_CPU_CORES: 4
+          JULIA_CPU_THREADS: 4
           JULIA_TEST_MAXRSS_MB: 800
     steps: &steps
       - run: | # install build dependencies
@@ -81,6 +81,6 @@ jobs:
       - image: circleci/python:2.7
         environment:
           ARCH: x86_64
-          JULIA_CPU_CORES: 4
+          JULIA_CPU_THREADS: 4
           JULIA_TEST_MAXRSS_MB: 800
     steps: *steps

.freebsdci.sh

@@ -41,7 +41,7 @@ runtests(){
     export FORCE_ASSERTIONS=1
     export LLVM_ASSERTIONS=1
     export JULIA_TEST_MAXRSS_MB=600
-    export JULIA_CPU_CORES=$MAKE_JOBS_NUMBER
+    export JULIA_CPU_THREADS=$MAKE_JOBS_NUMBER
 
     ./usr/bin/julia --check-bounds=yes test/runtests.jl all
     ./usr/bin/julia --check-bounds=yes test/runtests.jl \

.travis.yml

@@ -64,7 +64,7 @@ before_install:
         BUILDOPTS="-j5 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 USECCACHE=1";
         echo "override ARCH=$ARCH" >> Make.user;
         sudo sh -c "echo 0 > /proc/sys/net/ipv6/conf/lo/disable_ipv6";
-        export JULIA_CPU_CORES=4;
+        export JULIA_CPU_THREADS=4;
         export JULIA_TEST_MAXRSS_MB=1200;
         TESTSTORUN="all";
       elif [ `uname` = "Darwin" ]; then
@@ -90,7 +90,7 @@ before_install:
         export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$spawn_DYLD_FALLBACK_LIBRARY_PATH\" \$1";
         export BUILDOPTS="$BUILDOPTS spawn=\$(JULIA_MACOS_SPAWN)";
         make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make;
-        export JULIA_CPU_CORES=2;
+        export JULIA_CPU_THREADS=2;
         export JULIA_TEST_MAXRSS_MB=600;
         TESTSTORUN="all --skip linalg/triangular subarray"; fi # TODO: re enable these if possible without timing out
     - echo "override JULIA_CPU_TARGET=generic;native" >> Make.user

Makefile

@@ -581,7 +581,7 @@ testall: check-whitespace $(JULIA_BUILD_MODE)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
 testall1: check-whitespace $(JULIA_BUILD_MODE)
-	@env JULIA_CPU_CORES=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
+	@env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
 test-%: check-whitespace $(JULIA_BUILD_MODE)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)

NEWS.md

@@ -524,6 +524,11 @@ This section lists changes that do not have deprecation warnings.
   * `dot(u, v)` now acts recursively. Instead of `sum(u[i]' * v[i] for i in ...)`, it computes
     `sum(dot(u[i], v[i]) for i in ...)`, similarly to `vecdot` before ([#27401]).
 
+  * `Sys.CPU_CORES` has been renamed to `Sys.CPU_THREADS`; it still gives the number
+    of "logical cores" (including hyperthreading) rather than the number of physical
+    cores present on the CPU. Similarly, the environment variable `JULIA_CPU_CORES` is
+    deprecated in favor of `JULIA_CPU_THREADS` ([#27856]).
+
 Library improvements
 --------------------
 

README.windows.md

@@ -131,7 +131,7 @@ MinGW-w64 compilers available through Cygwin's package manager.
 
     3. Start the build
        ```sh
-       make -j 4   # Adjust the number of cores (4) to match your build environment.
+       make -j 4   # Adjust the number of threads (4) to match your build environment.
        ```
 
 

base/deprecated.jl

@@ -1575,6 +1575,10 @@ end
     Base.@deprecate log Base.log
 end
 
+# PR 27856
+@eval Base.Sys Base.@deprecate_binding CPU_CORES CPU_THREADS true nothing false
+# TODO: delete deprecation code in sysimg.jl and sysinfo.jl
+
 # PR 26071
 @deprecate(matchall(r::Regex, s::AbstractString; overlap::Bool = false),
            collect(m.match for m in eachmatch(r, s, overlap = overlap)))

base/sysimg.jl

@@ -457,12 +457,15 @@ function __init__()
     end
     # And try to prevent openblas from starting too many threads, unless/until specifically requested
     if !haskey(ENV, "OPENBLAS_NUM_THREADS") && !haskey(ENV, "OMP_NUM_THREADS")
-        cpu_cores = Sys.CPU_CORES::Int
-        if cpu_cores > 8 # always at most 8
+        cpu_threads = Sys.CPU_THREADS::Int
+        if cpu_threads > 8 # always at most 8
             ENV["OPENBLAS_NUM_THREADS"] = "8"
-        elseif haskey(ENV, "JULIA_CPU_CORES") # or exactly as specified
-            ENV["OPENBLAS_NUM_THREADS"] = cpu_cores
-        end # otherwise, trust that openblas will pick CPU_CORES anyways, without any intervention
+        elseif haskey(ENV, "JULIA_CPU_THREADS") # or exactly as specified
+            ENV["OPENBLAS_NUM_THREADS"] = cpu_threads
+        elseif haskey(ENV, "JULIA_CPU_CORES") # TODO: delete in 1.0 (deprecation)
+            Core.print("JULIA_CPU_CORES is deprecated, use JULIA_CPU_THREADS instead.\n")
+            ENV["OPENBLAS_NUM_THREADS"] = cpu_threads
+        end # otherwise, trust that openblas will pick CPU_THREADS anyways, without any intervention
     end
     # for the few uses of Libc.rand in Base:
     Libc.srand()

base/sysinfo.jl

@@ -7,7 +7,7 @@ Provide methods for retrieving information about hardware and the operating syst
 
 export BINDIR,
        STDLIB,
-       CPU_CORES,
+       CPU_THREADS,
        CPU_NAME,
        WORD_SIZE,
        ARCH,
@@ -47,15 +47,17 @@ STDLIB = "$BINDIR/../share/julia/stdlib/v$(VERSION.major).$(VERSION.minor)" # fo
 
 # helper to avoid triggering precompile warnings
 
-global CPU_CORES
 """
-    Sys.CPU_CORES
+    Sys.CPU_THREADS
 
-The number of logical CPU cores available in the system.
+The number of logical CPU cores available in the system, i.e. the number of threads
+that the CPU can run concurrently. Note that this is not necessarily the number of
+CPU cores, for example, in the presence of
+[hyper-threading](https://en.wikipedia.org/wiki/Hyper-threading).
 
-See the Hwloc.jl package for extended information, including number of physical cores.
+See Hwloc.jl or CpuId.jl for extended information, including number of physical cores.
 """
-:CPU_CORES
+CPU_THREADS = 1 # for bootstrap, changed on startup
 
 """
     Sys.ARCH
@@ -87,17 +89,24 @@ Standard word size on the current machine, in bits.
 const WORD_SIZE = Core.sizeof(Int) * 8
 
 function __init__()
-    env_cores = get(ENV, "JULIA_CPU_CORES", "")
-    global CPU_CORES = if !isempty(env_cores)
-        env_cores = tryparse(Int, env_cores)
-        if !(env_cores isa Int && env_cores > 0)
-            Core.print(Core.stderr, "WARNING: couldn't parse `JULIA_CPU_CORES` environment variable. Defaulting Sys.CPU_CORES to 1.\n")
-            env_cores = 1
+    env_threads = nothing
+    if haskey(ENV, "JULIA_CPU_THREADS")
+        env_threads = ENV["JULIA_CPU_THREADS"]
+    elseif haskey(ENV, "JULIA_CPU_CORES") # TODO: delete in 1.0 (deprecation)
+        Core.print("JULIA_CPU_CORES is deprecated, use JULIA_CPU_THREADS instead.\n")
+        env_threads = ENV["JULIA_CPU_CORES"]
+    end
+    global CPU_THREADS = if env_threads !== nothing
+        env_threads = tryparse(Int, env_threads)
+        if !(env_threads isa Int && env_threads > 0)
+            env_threads = Int(ccall(:jl_cpu_threads, Int32, ()))
+            Core.print(Core.stderr, "WARNING: couldn't parse `JULIA_CPU_THREADS` environment variable. Defaulting Sys.CPU_THREADS to $env_threads.\n")
         end
-        env_cores
+        env_threads
     else
-        Int(ccall(:jl_cpu_cores, Int32, ()))
+        Int(ccall(:jl_cpu_threads, Int32, ()))
     end
+    global CPU_CORES = CPU_THREADS # TODO: delete in 1.0 (deprecation)
     global SC_CLK_TCK = ccall(:jl_SC_CLK_TCK, Clong, ())
     global CPU_NAME = ccall(:jl_get_cpu_name, Ref{String}, ())
     global JIT = ccall(:jl_get_JIT, Ref{String}, ())

base/util.jl

@@ -730,7 +730,7 @@ kwdef_val(::Type{T}) where {T} = T()
 # testing
 
 """
-    Base.runtests(tests=["all"]; ncores=ceil(Int, Sys.CPU_CORES / 2),
+    Base.runtests(tests=["all"]; ncores=ceil(Int, Sys.CPU_THREADS / 2),
                   exit_on_error=false, [seed])
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
@@ -740,7 +740,7 @@ when `exit_on_error == true`.
 If a seed is provided via the keyword argument, it is used to seed the
 global RNG in the context where the tests are run; otherwise the seed is chosen randomly.
 """
-function runtests(tests = ["all"]; ncores = ceil(Int, Sys.CPU_CORES / 2),
+function runtests(tests = ["all"]; ncores = ceil(Int, Sys.CPU_THREADS / 2),
                   exit_on_error=false,
                   seed::Union{BitInteger,Nothing}=nothing)
     if isa(tests,AbstractString)
@@ -749,7 +749,7 @@ function runtests(tests = ["all"]; ncores = ceil(Int, Sys.CPU_CORES / 2),
     exit_on_error && push!(tests, "--exit-on-error")
     seed != nothing && push!(tests, "--seed=0x$(string(seed % UInt128, base=16))") # cast to UInt128 to avoid a minus sign
     ENV2 = copy(ENV)
-    ENV2["JULIA_CPU_CORES"] = "$ncores"
+    ENV2["JULIA_CPU_THREADS"] = "$ncores"
     try
         run(setenv(`$(julia_cmd()) $(joinpath(Sys.BINDIR::String,
             Base.DATAROOTDIR, "julia", "test", "runtests.jl")) $tests`, ENV2))

doc/src/base/constants.md

@@ -8,7 +8,7 @@ Base.C_NULL
 Base.VERSION
 Base.LOAD_PATH
 Base.Sys.BINDIR
-Base.Sys.CPU_CORES
+Base.Sys.CPU_THREADS
 Base.Sys.WORD_SIZE
 Base.Sys.KERNEL
 Base.Sys.ARCH

doc/src/manual/environment-variables.md

@@ -128,9 +128,9 @@ exists, or `emacs` otherwise.
 
 ## Parallelization
 
-### `JULIA_CPU_CORES`
+### `JULIA_CPU_THREADS`
 
-Overrides the global variable [`Base.Sys.CPU_CORES`](@ref), the number of
+Overrides the global variable [`Base.Sys.CPU_THREADS`](@ref), the number of
 logical CPU cores available.
 
 ### `JULIA_WORKER_TIMEOUT`

doc/src/manual/getting-started.md

@@ -102,7 +102,7 @@ julia [switches] -- [programfile] [args...]
 |`-e`, `--eval <expr>`                  |Evaluate `<expr>`|
 |`-E`, `--print <expr>`                 |Evaluate `<expr>` and display the result|
 |`-L`, `--load <file>`                  |Load `<file>` immediately on all processors|
-|`-p`, `--procs {N\|auto`}              |Integer value N launches N additional local worker processes; `auto` launches as many workers as the number of local cores|
+|`-p`, `--procs {N\|auto`}              |Integer value N launches N additional local worker processes; `auto` launches as many workers as the number of local CPU threads (logical cores)|
 |`--machine-file <file>`                |Run processes on hosts listed in `<file>`|
 |`-i`                                   |Interactive mode; REPL runs and `isinteractive()` is true|
 |`-q`, `--quiet`                        |Quiet startup: no banner, suppress REPL warnings|

doc/src/manual/parallel-computing.md

@@ -49,7 +49,7 @@ to as "workers". When there is only one process, process 1 is considered a worke
 workers are considered to be all processes other than process 1.
 
 Let's try this out. Starting with `julia -p n` provides `n` worker processes on the local machine.
-Generally it makes sense for `n` to equal the number of CPU cores on the machine. Note that the `-p`
+Generally it makes sense for `n` to equal the number of CPU threads (logical cores) on the machine. Note that the `-p`
 argument implicitly loads module `Distributed`.
 
 
@@ -1200,7 +1200,7 @@ would typically specify only `io` or `host` / `port`:
     workers.
 
       * `count` with an integer value `n` will launch a total of `n` workers.
-      * `count` with a value of `:auto` will launch as many workers as the number of cores on that machine.
+      * `count` with a value of `:auto` will launch as many workers as the number of CPU threads (logical cores) on that machine.
       * `exename` is the name of the `julia` executable including the full path.
       * `exeflags` should be set to the required command line arguments for new workers.
   * `tunnel`, `bind_addr`, `sshflags` and `max_parallel` are used when a ssh tunnel is required to

src/gc.c

@@ -2718,7 +2718,7 @@ void jl_gc_init(void)
 
 #ifdef _P64
     // on a big memory machine, set max_collect_interval to totalmem * nthreads / ncores / 2
-    size_t maxmem = (uv_get_total_memory() * jl_n_threads) / jl_cpu_cores() / 2;
+    size_t maxmem = (uv_get_total_memory() * jl_n_threads) / jl_cpu_threads() / 2;
     if (maxmem > max_collect_interval)
         max_collect_interval = maxmem;
 #endif

src/init.c

@@ -689,7 +689,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
 
 
 #if defined(__linux__)
-    int ncores = jl_cpu_cores();
+    int ncores = jl_cpu_threads();
     if (ncores > 1) {
         cpu_set_t cpumask;
         CPU_ZERO(&cpumask);

src/jloptions.c

@@ -93,7 +93,7 @@ static const char opts[]  =
 
     // parallel options
     " -p, --procs {N|auto}      Integer value N launches N additional local worker processes\n"
-    "                           \"auto\" launches as many workers as the number of local cores\n"
+    "                           \"auto\" launches as many workers as the number of local CPU threads (logical cores)\n"
     " --machine-file <file>     Run processes on hosts listed in <file>\n\n"
 
     // interactive options
@@ -385,7 +385,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
         case 'p': // procs
             errno = 0;
             if (!strcmp(optarg,"auto")) {
-                jl_options.nprocs = jl_cpu_cores();
+                jl_options.nprocs = jl_cpu_threads();
             }
             else {
                 long nprocs = strtol(optarg, &endptr, 10);

src/julia.h

@@ -1358,7 +1358,7 @@ JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, void *key,
 JL_DLLEXPORT int jl_errno(void);
 JL_DLLEXPORT void jl_set_errno(int e);
 JL_DLLEXPORT int32_t jl_stat(const char *path, char *statbuf);
-JL_DLLEXPORT int jl_cpu_cores(void);
+JL_DLLEXPORT int jl_cpu_threads(void);
 JL_DLLEXPORT long jl_getpagesize(void);
 JL_DLLEXPORT long jl_getallocationgranularity(void);
 JL_DLLEXPORT int jl_is_debugbuild(void);

src/sys.c

@@ -353,7 +353,7 @@ JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n)
 JL_DLLEXPORT int jl_errno(void) { return errno; }
 JL_DLLEXPORT void jl_set_errno(int e) { errno = e; }
 
-// -- get the number of CPU cores --
+// -- get the number of CPU threads (logical cores) --
 
 #ifdef _OS_WINDOWS_
 typedef DWORD (WINAPI *GAPC)(WORD);
@@ -362,7 +362,7 @@ typedef DWORD (WINAPI *GAPC)(WORD);
 #endif
 #endif
 
-JL_DLLEXPORT int jl_cpu_cores(void)
+JL_DLLEXPORT int jl_cpu_threads(void)
 {
 #if defined(HW_AVAILCPU) && defined(HW_NCPU)
     size_t len = 4;

src/threading.c

@@ -555,7 +555,7 @@ void jl_init_threading(void)
 #endif
 
     // how many threads available, usable
-    int max_threads = jl_cpu_cores();
+    int max_threads = jl_cpu_threads();
     jl_n_threads = JULIA_NUM_THREADS;
     cp = getenv(NUM_THREADS_NAME);
     if (cp) {

stdlib/Distributed/src/cluster.jl

@@ -452,7 +452,7 @@ function setup_launched_worker(manager, wconfig, launched_q)
     # same type. This is done by setting an appropriate value to `WorkerConfig.cnt`.
     cnt = something(wconfig.count, 1)
     if cnt === :auto
-        cnt = wconfig.environ[:cpu_cores]
+        cnt = wconfig.environ[:cpu_threads]
     end
     cnt = cnt - 1   # Removing self from the requested number
 

stdlib/Distributed/src/managers.jl

@@ -64,7 +64,7 @@ specified, other workers will connect to this worker at the specified `bind_addr
 `port`.
 
 `count` is the number of workers to be launched on the specified host. If specified as
-`:auto` it will launch as many workers as the number of cores on the specific host.
+`:auto` it will launch as many workers as the number of CPU threads on the specific host.
 
 Keyword arguments:
 
@@ -294,13 +294,13 @@ end
 """
     addprocs(; kwargs...) -> List of process identifiers
 
-Equivalent to `addprocs(Sys.CPU_CORES; kwargs...)`
+Equivalent to `addprocs(Sys.CPU_THREADS; kwargs...)`
 
 Note that workers do not run a `.julia/config/startup.jl` startup script, nor do they synchronize
 their global state (such as global variables, new method definitions, and loaded modules) with any
 of the other running processes.
 """
-addprocs(; kwargs...) = addprocs(Sys.CPU_CORES; kwargs...)
+addprocs(; kwargs...) = addprocs(Sys.CPU_THREADS; kwargs...)
 
 """
     addprocs(np::Integer; restrict=true, kwargs...) -> List of process identifiers

stdlib/Distributed/src/messages.jl

@@ -71,7 +71,7 @@ struct JoinPGRPMsg <: AbstractMsg
     lazy::Bool
 end
 struct JoinCompleteMsg <: AbstractMsg
-    cpu_cores::Int
+    cpu_threads::Int
     ospid::Int
 end
 

stdlib/Distributed/src/process_messages.jl

@@ -330,7 +330,7 @@ function handle_msg(msg::JoinPGRPMsg, header, r_stream, w_stream, version)
     for wt in wait_tasks; Base._wait(wt); end
 
     send_connection_hdr(controller, false)
-    send_msg_now(controller, MsgHeader(RRID(0,0), header.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid()))
+    send_msg_now(controller, MsgHeader(RRID(0,0), header.notify_oid), JoinCompleteMsg(Sys.CPU_THREADS, getpid()))
 end
 
 function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConfig)
@@ -349,7 +349,7 @@ end
 function handle_msg(msg::JoinCompleteMsg, header, r_stream, w_stream, version)
     w = map_sock_wrkr[r_stream]
     environ = something(w.config.environ, Dict())
-    environ[:cpu_cores] = msg.cpu_cores
+    environ[:cpu_threads] = msg.cpu_threads
     w.config.environ = environ
     w.config.ospid = msg.ospid
     w.version = version

stdlib/Distributed/test/distributed_exec.jl

@@ -675,7 +675,7 @@ if Sys.isunix() # aka have ssh
 
     print("\nMixed ssh addprocs with :auto\n")
     new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags)
-    @test length(new_pids) == (2 + Sys.CPU_CORES)
+    @test length(new_pids) == (2 + Sys.CPU_THREADS)
     test_n_remove_pids(new_pids)
 
     print("\nMixed ssh addprocs with numeric counts\n")

stdlib/InteractiveUtils/test/runtests.jl

@@ -192,7 +192,7 @@ end
         @test !occursin("Environment:", read(setenv(`$exename -e 'using InteractiveUtils; versioninfo()'`,
                                                     String[]), String))
         @test  occursin("Environment:", read(setenv(`$exename -e 'using InteractiveUtils; versioninfo()'`,
-                                                    String["JULIA_CPU_CORES=1"]), String))
+                                                    String["JULIA_CPU_THREADS=1"]), String))
     end
 end