Move the test runner to use TestItems.jl

test/Project.toml

@@ -22,8 +22,8 @@ QuantumOpticsBase = "4f57444f-1401-5e15-980d-4471b28d5678"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
 SIMD = "fdea26ae-647d-5447-a871-4b548cad5224"
-SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Static = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
 StridedViews = "4db3bf67-4bd7-4b4e-b153-31dc3fb37143"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"

test/runtests.jl

@@ -1,76 +1,28 @@
-using SafeTestsets
 using QuantumClifford
+using TestItemRunner
 
-function doset(descr)
-    if length(ARGS) == 0
-        return true
-    end
-    for a in ARGS
-        if occursin(lowercase(a), lowercase(descr))
-            return true
-        end
-    end
-    return false
+if get(ENV, "GPU_TESTS", "") != "true"
+    println("skipping gpu tests (set GPU_TESTS=true to test gpu)")
 end
 
-macro doset(descr)
-    quote
-        if doset($descr)
-            @safetestset $descr begin
-                include("test_"*$descr*".jl")
+# filter for the test
+testfilter = ti -> begin
+  exclude = Symbol[]
+  if get(ENV,"JET_TEST","")!="true"
+    push!(exclude, :jet)
   end
+  if !(VERSION >= v"1.10")
+    push!(exclude, :doctests)
+    push!(exclude, :aqua)
   end
+
+  if get(ENV, "GPU_TESTS", "")!="true"
+    push!(exclude, :gpu)
   end
+
+  return all(!in(exclude), ti.tags)
 end
 
 println("Starting tests with $(Threads.nthreads()) threads out of `Sys.CPU_THREADS = $(Sys.CPU_THREADS)`...")
 
-
-# in order to run the gpu tests automatically set GPU_TESTS to true in the .env file
-if get(ENV, "GPU_TESTS", "") == "true"
-    @doset "gpu"
-else
-    println("skipping gpu tests (set GPU_TESTS=true to test gpu)")
-end
-
-@doset "throws"
-@doset "paulis"
-@doset "stabs"
-@doset "stabcanon"
-@doset "mul_leftright"
-@doset "inner"
-@doset "embed"
-@doset "gf2"
-@doset "projections"
-@doset "expect"
-@doset "trace"
-@doset "cliff"
-@doset "symcliff"
-@doset "symcontrolled"
-@doset "classicalreg"
-@doset "random"
-@doset "noisycircuits"
-@doset "syndromemeas"
-@doset "bitpack"
-@doset "memorylayout"
-@doset "graphs"
-@doset "hash"
-@doset "entanglement"
-@doset "enumerate"
-@doset "quantumoptics"
-@doset "ecc"
-@doset "ecc_codeproperties"
-@doset "ecc_decoder_all_setups"
-@doset "ecc_encoding"
-@doset "ecc_gottesman"
-@doset "ecc_reedmuller"
-@doset "ecc_bch"
-@doset "ecc_syndromes"
-@doset "ecc_throws"
-@doset "precompile"
-@doset "pauliframe"
-@doset "sumtypecompactification"
-@doset "allocations"
-VERSION >= v"1.10" && @doset "doctests"
-get(ENV,"JET_TEST","")=="true" && @doset "jet"
-VERSION >= v"1.10" && @doset "aqua"
+@run_package_tests filter=testfilter

test/test_allocations.jl

@@ -1,7 +1,5 @@
-using QuantumClifford
+@testitem "Allocation checks" begin
     using QuantumClifford: mul_left!
-
-@testset "Allocation checks" begin
     n = Threads.nthreads()
     allocated(f::F) where {F} = @allocated f()
     @testset "apply! mul_left! canonicalize!" begin

test/test_aqua.jl

@@ -1,3 +1,4 @@
+@testitem "Aqua" tags=[:aqua] begin
     using Aqua
-using QuantumClifford
     Aqua.test_all(QuantumClifford)
+end

test/test_bitpack.jl

@@ -1,8 +1,8 @@
+@testitem "Alternative bit packing" begin
     using Random
-using QuantumClifford
     using QuantumClifford: Tableau
 
-@testset "Alternative bit packing" begin
+    @testset "alternative bit packing" begin
         for n in [1,3] # can not go higher than 4 (limitation from SIMD acting on transposed/strided arrays)
             N = 64*n-2
             s64 = random_stabilizer(N,N);
@@ -49,3 +49,4 @@ using QuantumClifford: Tableau
             end
         end
     end
+end

test/test_classicalreg.jl

@@ -1,6 +1,5 @@
+@testitem "Classical" begin
     using Random
-using QuantumClifford
-
     using QuantumClifford: stab_looks_good, destab_looks_good, mixed_stab_looks_good, mixed_destab_looks_good
 
     n=5
@@ -25,3 +24,4 @@ for state in [mdstab,reg,regmd]
         end
     end
     @test tab(canonicalize!(stabilizerview(reg))).xzs == tab(canonicalize!(stabilizerview(mdstab))).xzs
+end

test/test_cliff.jl

@@ -1,6 +1,5 @@
+@testitem "Clifford" begin
     using Random
-using QuantumClifford
-
     using QuantumClifford: stab_looks_good, destab_looks_good, mixed_stab_looks_good, mixed_destab_looks_good
     using QuantumClifford: mul_left!
 
@@ -105,3 +104,4 @@ end
             end
         end
     end
+end

test/test_doctests.jl

@@ -1,11 +1,14 @@
+@testitem "Doctests" tags=[:doctests] begin
     using Documenter
     using QuantumClifford
 
     ENV["LINES"] = 80    # for forcing `displaysize(io)` to be big enough
     ENV["COLUMNS"] = 80
-@testset "Doctests" begin
     DocMeta.setdocmeta!(QuantumClifford, :DocTestSetup, :(using QuantumClifford; using QuantumClifford.ECC); recursive=true)
-    doctest(QuantumClifford,
+    doctestfilters = [r"(QuantumClifford\.|)"]
+    doctest(QuantumClifford;
+            doctestfilters
             #fix=true
            )
+    # TODO failures in VSCode related to https://github.com/julia-vscode/TestItemRunner.jl/issues/49
 end

test/test_ecc.jl

@@ -1,5 +1,4 @@
-using Test
-using QuantumClifford
+@testitem "ECC" begin
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC
 
@@ -70,3 +69,4 @@ end
         @test isdegenerate(Steane7(), 2) == true
         @test isdegenerate(Bitflip3()) == true
     end
+end

test/test_ecc_bch.jl

@@ -1,6 +1,5 @@
-using Test
+@testitem "ECC BCH" begin
     using LinearAlgebra
-using QuantumClifford
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC, BCH, generator_polynomial
     using Nemo: ZZ, residue_ring, matrix, finite_field, GF, minpoly, coeff, lcm, FqPolyRingElem, FqFieldElem, is_zero, degree, defining_polynomial, is_irreducible 
@@ -83,3 +82,4 @@ end
             @test check_designed_distance(parity_checks(BCH(m, t)), t) == true
         end
     end
+end

test/test_ecc_codeproperties.jl

@@ -1,5 +1,4 @@
-using Test
-using QuantumClifford
+@testitem "ECC code properties" begin
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC
 
@@ -38,3 +37,4 @@ end
             @test QuantumClifford.stab_looks_good(copy(H))
         end
     end
+end

test/test_ecc_decoder_all_setups.jl

@@ -1,5 +1,4 @@
-using Test
-using QuantumClifford
+@testitem "ECC Decoder" begin
     using QuantumClifford.ECC
 
     import PyQDecoders
@@ -96,3 +95,4 @@ import LDPCDecoders
             end
         end
     end
+end

test/test_ecc_encoding.jl

@@ -1,10 +1,10 @@
-using Test
+@testitem "encoding circuits - compare to algebraic construction of encoded state" begin
     using QuantumClifford
     using QuantumClifford.ECC
 
     include("test_ecc_base.jl")
 
-@testset "encoding circuits - compare to algebraic construction of encoded state" begin
+
     # This test verifies that logical measurements on an encoded state match the physical pre-encoded state.
     # This test skips verifying the permutations of qubits during canonicalization are properly undone,
     # i.e. we modify the code we are testing so that the canonicalization does not need any permutations.

test/test_ecc_gottesman.jl

@@ -1,11 +1,8 @@
-using Test
-using QuantumClifford
+@testitem "Gottesman codes should correct all single-qubit errors" begin
     using QuantumClifford: mul_left!
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC
 
-
-@testset "Gottesman codes should correct all single-qubit errors" begin
     for j in 3:12
         H = parity_checks(Gottesman(j))
         syndromes = Set([]) # the set automatically removes repeated entries

test/test_ecc_reedmuller.jl

@@ -1,8 +1,7 @@
-using Test
+@testitem "Reed-Muller" begin
     using Nemo
     using Combinatorics
     using LinearAlgebra
-using QuantumClifford
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC, ReedMuller
 
@@ -66,3 +65,4 @@ end
                                                  1 0 1 0 0 0 0 0 1 0 1 0 0 0 0 0;
                                                  1 0 0 0 1 0 0 0 1 0 0 0 1 0 0 0]
     end
+end

test/test_ecc_syndromes.jl

@@ -1,5 +1,4 @@
-using Test
-using QuantumClifford
+@testitem "ECC Syndromes" begin
     using QuantumClifford: mul_left!, embed
     using QuantumClifford.ECC
     using QuantumClifford.ECC: AbstractECC
@@ -47,3 +46,4 @@ end
             pframe_naive_vs_shor_syndrome(c)
         end
     end
+end

test/test_ecc_throws.jl

@@ -1,9 +1,9 @@
-using Test
-using QuantumClifford
+@testitem "ECC throws" begin
     using QuantumClifford.ECC: ReedMuller, BCH
 
     @test_throws ArgumentError ReedMuller(-1, 3)
     @test_throws ArgumentError ReedMuller(1, 0)
 
     @test_throws ArgumentError BCH(2, 2)
     @test_throws ArgumentError BCH(3, 4)
+end

test/test_embed.jl

@@ -1,7 +1,4 @@
-using Test
-using QuantumClifford
-
-@testset "embed PauliOperator" begin
+@testitem "embed PauliOperator" begin
     @test embed(5,3,P"-Y") == P"-__Y__"
     @test embed(5,(3,5),P"-YZ") == P"-__Y_Z"
     @test embed(5,[3,5],P"-YZ") == P"-__Y_Z"

test/test_entanglement.jl

@@ -1,7 +1,5 @@
-using Test
-
+@testitem "Entanglement" begin
     using Graphs
-using QuantumClifford
 
     test_sizes = [1,2,10,63,64,65,127,128,129] # Including sizes that would test off-by-one errors in the bit encoding.
 
@@ -52,3 +50,4 @@ end
         @test entanglement_entropy(copy(s), subsystem, Val(:graph))==2
         @test entanglement_entropy(copy(s), subsystem, Val(:rref))==2
     end
+end

test/test_enumerate.jl

@@ -1,6 +1,4 @@
-using QuantumClifford
-
-@testset "Enumeration" begin
+@testitem "Enumeration" begin
     @test Set([copy(op) for op in enumerate_cliffords(1)]) == Set((random_clifford(1,phases=false) for i in 1:70))
     @test Set([copy(op) for op in enumerate_phases(enumerate_cliffords(1))]) == Set((random_clifford(1,phases=true) for i in 1:300))
     @test length(collect(enumerate_cliffords(2))) == length(collect(enumerate_phases(enumerate_cliffords(2))))/2^4 == 720

test/test_expect.jl

@@ -1,5 +1,4 @@
-using QuantumClifford
-
+@testite "Expectation of Pauli strings on stabilizer states" begin
     using QuantumClifford: stab_looks_good, destab_looks_good, mixed_stab_looks_good, mixed_destab_looks_good
     using QuantumClifford: projectremoverand!
 
@@ -29,7 +28,6 @@ function alter_expect(p::PauliOperator, s::Stabilizer)
         result === 0x03 && return -im
     end
 
-@testset "Expectation of Pauli strings on stabilizer states" begin
     st = bell()
     apply!(st, sX(2))
     @test expect(P"XX", st) == alter_expect(P"XX", st) == 1

test/test_gf2.jl

@@ -1,12 +1,8 @@
+@testitem "GF(2) representations" begin
     using Random
-using QuantumClifford
-
     using QuantumClifford: stab_looks_good, destab_looks_good, mixed_stab_looks_good, mixed_destab_looks_good
-
     test_sizes = [1,2,10,63,64,65,127,128,129] # Including sizes that would test off-by-one errors in the bit encoding.
 
-function test_gf2()
-    @testset "GF(2) representations" begin
     @testset "Equivalence of GF(2) Gaussian elimination and Stabilizer canonicalization" begin
         for n in test_sizes
             for rep in 1:5
@@ -29,6 +25,3 @@ function test_gf2()
         end
     end
 end
-end
-
-test_gf2()

test/test_gpu.jl

@@ -1,5 +1,4 @@
-using Test
-using QuantumClifford
+@testitem "GPU" tags=[:gpu] begin
     using QuantumClifford: to_cpu, to_gpu
     using CUDA
 
@@ -145,3 +144,4 @@ end
             approx(vec(avg_result), [0, .5, 2p/3, .5], error_threshold)
         end
     end
+end

test/test_graphs.jl

@@ -1,10 +1,9 @@
+@testitem "Graph states" begin
     using Graphs
     using Random
-using QuantumClifford
 
     test_sizes = [1,2,10,63,64,65,127,128,129] # Including sizes that would test off-by-one errors in the bit encoding.
 
-@testset "Graph states" begin
     for n in [1,test_sizes...]
         s = random_stabilizer(n)
         g, h_idx, ip_idx, z_idx = graphstate(s)

test/test_hash.jl

@@ -1,6 +1,4 @@
-using QuantumClifford
-
-@testset "Hashing" begin
+@testitem "Hashing" begin
     @test hash(P"X") == hash(P"X")
     @test hash(S"X") == hash(S"X")
     @test hash(C"X Z") == hash(C"X Z")