allow stdlibs to run their tests in parallel (#25778)

* allow stdlibs to run their tests in parallel

this is done by creating a file `tests` in the `test` folder
that lists the different files with tests (excluding ".jl").
If this file does not exist, we fall back to using the runtests.jl file

* fix runtest files for stdlibs

stdlib/Dates/test/runtests.jl

@@ -1,17 +1,8 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
 module DateTests
 
-using Test
-using Dates
-
-include("accessors.jl")
-include("adjusters.jl")
-include("query.jl")
-include("periods.jl")
-include("ranges.jl")
-include("rounding.jl")
-include("types.jl")
-include("io.jl")
-include("arithmetic.jl")
-include("conversions.jl")
+for file in readlines(joinpath(@__DIR__, "testgroups"))
+    include(file * ".jl")
+end
 
 end

stdlib/Dates/test/testgroups

@@ -0,0 +1,10 @@
+accessors
+adjusters
+query
+periods
+ranges
+rounding
+types
+io
+arithmetic
+conversions

stdlib/LinearAlgebra/test/runtests.jl

@@ -1,24 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-include("triangular.jl")
-include("qr.jl")
-include("dense.jl")
-include("matmul.jl")
-include("schur.jl")
-include("special.jl")
-include("eigen.jl")
-include("bunchkaufman.jl")
-include("svd.jl")
-include("lapack.jl")
-include("tridiag.jl")
-include("bidiag.jl")
-include("diagonal.jl")
-include("cholesky.jl")
-include("lu.jl")
-include("symmetric.jl")
-include("generic.jl")
-include("uniformscaling.jl")
-include("lq.jl")
-include("hessenberg.jl")
-include("blas.jl")
-include("adjtrans.jl")
+for file in readlines(joinpath(@__DIR__, "testgroups"))
+    include(file * ".jl")
+end

stdlib/LinearAlgebra/test/testgroups

@@ -0,0 +1,22 @@
+triangular
+qr
+dense
+matmul
+schur
+special
+eigen
+bunchkaufman
+svd
+lapack
+tridiag
+bidiag
+diagonal
+cholesky
+lu
+symmetric
+generic
+uniformscaling
+lq
+hessenberg
+blas
+adjtrans

stdlib/SparseArrays/test/higherorderfns.jl

@@ -4,6 +4,11 @@
 # base/sparse/higherorderfns.jl, particularly map[!]/broadcast[!] for SparseVectors and
 # SparseMatrixCSCs at present.
 
+module HigherOrderFnsTests
+
+using Test
+using SparseArrays
+using LinearAlgebra
 using Random
 
 @testset "map[!] implementation specialized for a single (input) sparse vector/matrix" begin
@@ -548,3 +553,5 @@ end
     @test spzeros(1,2) .+ spzeros(0,1) == zeros(0,2)
     @test spzeros(1,2) .* spzeros(0,1) == zeros(0,2)
 end
+
+end # module

stdlib/SparseArrays/test/runtests.jl

@@ -1,8 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-using Test, SparseArrays
-using LinearAlgebra
-
-include("higherorderfns.jl")
-include("sparse.jl")
-include("sparsevector.jl")
+for file in readlines(joinpath(@__DIR__, "testgroups"))
+    include(file * ".jl")
+end

stdlib/SparseArrays/test/sparse.jl

@@ -1,6 +1,10 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-using LinearAlgebra: mul!, ldiv!, rdiv!
+module SparseTests
+
+using Test
+using SparseArrays
+using LinearAlgebra
 using Base.Printf: @printf
 using Random
 
@@ -2208,4 +2212,6 @@ end
 @testset "findnz on non-sparse arrays" begin
     @test findnz([0 1; 0 2]) == ([1, 2], [2, 2], [1, 2])
     @test findnz(BitArray([false true; false true])) == ([1, 2], [2, 2], trues(2))
-end
+end
+
+end # module

stdlib/SparseArrays/test/sparsevector.jl

@@ -1,5 +1,9 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+module SparseVectorTests
+
+using Test
+using SparseArrays
 using LinearAlgebra
 using Random
 
@@ -1264,3 +1268,5 @@ end
               LinearAlgebra.lowrankupdate!(Matrix(1.0*I, n, n), fill(1.0, n), Ajview)
     end
 end
+
+end # module

stdlib/SparseArrays/test/testgroups

@@ -0,0 +1,3 @@
+higherorderfns
+sparse
+sparsevector

test/choosetests.jl

@@ -146,6 +146,20 @@ function choosetests(choices = [])
         prepend!(tests, STDLIBS)
     end
 
+    new_tests = String[]
+    for test in tests
+        if test in STDLIBS
+            testfile = joinpath(STDLIB_DIR, test, "test", "testgroups")
+            if isfile(testfile)
+                prepend!(new_tests, (test * "/") .* readlines(testfile))
+            else
+                push!(new_tests, test)
+            end
+        end
+    end
+    filter!(x -> (x != "stdlib" && !(x in STDLIBS)) , tests)
+    prepend!(tests, new_tests)
+
     # do ambiguous first to avoid failing if ambiguities are introduced by other tests
     if "ambiguous" in skip_tests
         filter!(x -> x != "ambiguous", tests)

test/runtests.jl

@@ -16,18 +16,24 @@ const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
 else
     typemax(Csize_t)
 end
+limited_worker_rss = max_worker_rss != typemax(Csize_t)
 
 function test_path(test)
-    if test in STDLIBS
-        return joinpath(STDLIB_DIR, test, "test", "runtests")
+    t = split(test, '/')
+    if t[1] in STDLIBS
+        if length(t) == 2
+            return joinpath(STDLIB_DIR, t[1], "test", t[2])
+        else
+            return joinpath(STDLIB_DIR, t[1], "test", "runtests")
+        end
     else
         return joinpath(@__DIR__, test)
     end
 end
 
 # Check all test files exist
 isfiles = isfile.(test_path.(tests) .* ".jl")
-if any(equalto(false), isfiles)
+if !all(isfiles)
     error("did not find test files for the following tests: ",
           join(tests[.!(isfiles)], ", "))
 end
@@ -46,7 +52,7 @@ move_to_node1("SharedArrays")
 
 # In a constrained memory environment, run the "distributed" test after all other tests
 # since it starts a lot of workers and can easily exceed the maximum memory
-max_worker_rss != typemax(Csize_t) && move_to_node1("Distributed")
+limited_worker_rss && move_to_node1("Distributed")
 
 import LinearAlgebra
 cd(dirname(@__FILE__)) do
@@ -61,34 +67,38 @@ cd(dirname(@__FILE__)) do
     @everywhere include("testdefs.jl")
 
     #pretty print the information about gc and mem usage
-    name_align    = maximum([length("Test (Worker)"); map(x -> length(x) + 3 + ndigits(nworkers()), tests)])
+    testgroupheader = "Test"
+    workerheader = "(Worker)"
+    name_align    = maximum([length(testgroupheader) + length(" ") + length(workerheader); map(x -> length(x) + 3 + ndigits(nworkers()), tests)])
     elapsed_align = length("Time (s)")
     gc_align      = length("GC (s)")
     percent_align = length("GC %")
     alloc_align   = length("Alloc (MB)")
     rss_align     = length("RSS (MB)")
-    printstyled(rpad("Test (Worker)", name_align, " "), " | ", color=:white)
+    printstyled(testgroupheader, color=:white)
+    printstyled(lpad(workerheader, name_align - length(testgroupheader) + 1), " | ", color=:white)
     printstyled("Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB)\n", color=:white)
     results=[]
     print_lock = ReentrantLock()
 
     function print_testworker_stats(test, wrkr, resp)
         lock(print_lock)
         try
-            printstyled(rpad(test*" ($wrkr)", name_align, " "), " | ", color=:white)
+            printstyled(test, color=:white)
+            printstyled(lpad("($wrkr)", name_align - length(test) + 1, " "), " | ", color=:white)
             time_str = @sprintf("%7.2f",resp[2])
-            printstyled(rpad(time_str,elapsed_align," "), " | ", color=:white)
-            gc_str = @sprintf("%5.2f",resp[5].total_time/10^9)
-            printstyled(rpad(gc_str, gc_align, " "), " | ", color=:white)
+            printstyled(lpad(time_str, elapsed_align, " "), " | ", color=:white)
+            gc_str = @sprintf("%5.2f", resp[5].total_time / 10^9)
+            printstyled(lpad(gc_str, gc_align, " "), " | ", color=:white)
 
             # since there may be quite a few digits in the percentage,
             # the left-padding here is less to make sure everything fits
-            percent_str = @sprintf("%4.1f",100*resp[5].total_time/(10^9*resp[2]))
-            printstyled(rpad(percent_str, percent_align, " "), " | ", color=:white)
-            alloc_str = @sprintf("%5.2f",resp[3]/2^20)
-            printstyled(rpad(alloc_str, alloc_align, " "), " | ", color=:white)
-            rss_str = @sprintf("%5.2f",resp[6]/2^20)
-            printstyled(rpad(rss_str, rss_align, " "), "\n", color=:white)
+            percent_str = @sprintf("%4.1f", 100 * resp[5].total_time / (10^9 * resp[2]))
+            printstyled(lpad(percent_str, percent_align, " "), " | ", color=:white)
+            alloc_str = @sprintf("%5.2f", resp[3] / 2^20)
+            printstyled(lpad(alloc_str, alloc_align, " "), " | ", color=:white)
+            rss_str = @sprintf("%5.2f", resp[6] / 2^20)
+            printstyled(lpad(rss_str, rss_align, " "), "\n", color=:white)
         finally
             unlock(print_lock)
         end