Update testing

test/common.jl

@@ -3,31 +3,45 @@ using Chmy
 
 using KernelAbstractions
 
-# testing for various floating point arithmetic precisions
-precisions = [Float32, Float64]
+compatible(::Backend, ::DataType) = true
 
-# add KA backends
-backends = KernelAbstractions.Backend[CPU()]
+# number types to test
+TEST_TYPES = [Float32, Float64]
 
-# do not test Float64 on Metal.jl
-skip_Float64 = [false]
+# add backends
+TEST_BACKENDS = []
 
-if get(ENV, "JULIA_CHMY_BACKEND", "") == "AMDGPU"
-    using AMDGPU
-    if AMDGPU.functional()
-        push!(backends, ROCBackend())
-        push!(skip_Float64, false)
-    end
-elseif get(ENV, "JULIA_CHMY_BACKEND", "") == "CUDA"
+if haskey(ENV, "JULIA_CHMY_BACKEND_CPU")
+    push!(TEST_BACKENDS, CPU())
+end
+
+if haskey(ENV, "JULIA_CHMY_BACKEND_CUDA")
     using CUDA
     if CUDA.functional()
-        push!(backends, CUDABackend())
-        push!(skip_Float64, false)
+        push!(TEST_BACKENDS, CUDABackend())
     end
-elseif get(ENV, "JULIA_CHMY_BACKEND", "") == "Metal"
+end
+
+if haskey(ENV, "JULIA_CHMY_BACKEND_AMDGPU")
+    using AMDGPU
+    if AMDGPU.functional()
+        push!(TEST_BACKENDS, ROCBackend())
+    end
+end
+
+if haskey(ENV, "JULIA_CHMY_BACKEND_Metal")
     using Metal
+
+    function compatible(::MetalBackend, T::DataType)
+        try
+            Metal.check_eltype(T)
+            return true
+        catch
+            return false
+        end
+    end
+
     if Metal.functional()
-        push!(backends, MetalBackend())
-        push!(skip_Float64, true)
+        push!(TEST_BACKENDS, MetalBackend())
     end
 end

test/runtests.jl

@@ -3,54 +3,58 @@ using Chmy
 
 using Pkg
 
-excludedfiles = ["test_excluded.jl"]
-
 # distributed
 using MPI
 
-function parse_flags!(args, flag; default=nothing, typ=typeof(default))
-    for f in args
-        startswith(f, flag) || continue
-
-        if f != flag
-            val = split(f, '=')[2]
-            if !(typ ≡ nothing || typ <: AbstractString)
-                @show typ val
-                val = parse(typ, val)
-            end
-        else
-            val = default
-        end
+EXCLUDE_TESTS = []
+
+istest(f) = startswith(f, "test_") && endswith(f, ".jl")
+
+function parse_flag(args, flag; default=nothing, type::DataType=Nothing)
+    key = findfirst(arg -> startswith(arg, flag), args)
+
+    if isnothing(key)
+        # flag not found
+        return false, default
+    elseif args[key] == flag
+        # flag found but no value
+        return true, default
+    end
+
+    splitarg = split(args[key], '=')
 
-        filter!(x -> x != f, args)
-        return true, val
+    if splitarg[1] != flag
+        # argument started with flag but is not the flag
+        return false, default
     end
-    return false, default
+
+    values = strip.(split(splitarg[2], ','))
+
+    if type <: Nothing || type <: AbstractString
+        # common cases, return as strings
+        return true, values
+    elseif !(type <: Number) || !isbitstype(type)
+        error("type must be a bitstype number, got '$type'")
+    end
+
+    return true, parse.(Ref(type), values)
 end
 
 function runtests()
-    testdir   = pwd()
-    istest(f) = endswith(f, ".jl") && startswith(basename(f), "test_")
-    testfiles = sort(filter(istest, vcat([joinpath.(root, files) for (root, dirs, files) in walkdir(testdir)]...)))
+    testdir   = @__DIR__
+    testfiles = sort(filter(istest, readdir(testdir)))
 
     nfail = 0
     printstyled("Testing package Chmy.jl\n"; bold=true, color=:white)
 
     for f in testfiles
         println("")
-        if basename(f) ∈ excludedfiles
-            println("Test Skip:")
-            println("$f")
+        if f ∈ EXCLUDE_TESTS
+            @info "Skip test:" f
             continue
         end
         try
-            # if basename(f) ∈ test_distributed
-            #     nprocs = contains(f, "2D") ? nprocs_2D : nprocs_3D
-            #     cmd(n=nprocs) = `$(mpiexec()) -n $n $(Base.julia_cmd()) --startup-file=no --color=yes $(joinpath(testdir, f))`
-            #     run(cmd())
-            # else
-                run(`$(Base.julia_cmd()) --startup-file=no $(joinpath(testdir, f))`)
-            # end
+            run(`$(Base.julia_cmd()) --startup-file=no $(joinpath(testdir, f))`)
         catch ex
             @error ex
             nfail += 1
@@ -59,17 +63,13 @@ function runtests()
     return nfail
 end
 
-_, backend_name = parse_flags!(ARGS, "--backend"; default="CPU", typ=String)
-
-@static if backend_name == "AMDGPU"
-    Pkg.add("AMDGPU")
-    ENV["JULIA_CHMY_BACKEND"] = "AMDGPU"
-elseif backend_name == "CUDA"
-    Pkg.add("CUDA")
-    ENV["JULIA_CHMY_BACKEND"] = "CUDA"
-elseif backend_name == "Metal"
-    Pkg.add("Metal")
-    ENV["JULIA_CHMY_BACKEND"] = "Metal"
+_, backends = parse_flag(ARGS, "--backends"; default=["CPU"])
+
+for backend in backends
+    if backend != "CPU"
+        Pkg.add(backend)
+    end
+    ENV["JULIA_CHMY_BACKEND_$backend"] = true
 end
 
 exit(runtests())

test/test_architectures.jl

@@ -2,7 +2,7 @@ include("common.jl")
 
 using Chmy.Architectures
 
-for backend in backends
+for backend in TEST_BACKENDS
     @testset "$(basename(@__FILE__)) (backend: $backend)" begin
         device = get_device(backend, 1)
         arch = SingleDeviceArchitecture(backend, device)