Reorganise failing tests

 - Disabled reference tests 25 and 30, which require StatsPlots support,
   for all backends (in PlotsBase/src/examples.jl)
 - Separate tests that are skipped and tests that are known to be broken
   due to upstream issues
 - Re-enable reference test 41 as upstream issue has been fixed
   (JuliaLang/julia#47261)
 - Disable reference test 50 because of upstream issue
   (jheinen/GR.jl#550)

PlotsBase/src/examples.jl

@@ -1258,12 +1258,10 @@ _animation_examples = [02, 31]
 _backend_skips = Dict(
     :none => Int[],
     :pythonplot => Int[],
-    :gr => [25, 30],  # TODO: add back when StatsPlots is available
+    :gr => Int[],
     :plotlyjs => [
         21,
         24,
-        25,
-        30,
         49,
         50,
         51,
@@ -1314,6 +1312,10 @@ _backend_skips = Dict(
     ],
 )
 _backend_skips[:plotly] = _backend_skips[:plotlyjs]
+# 25 and 30 require StatsPlots, which doesn't support Plots v2 yet
+for backend in keys(_backend_skips)
+    append!(_backend_skips[backend], [25, 30])
+end
 
 # ---------------------------------------------------------------------------------
 # replace `f(args...)` with `f(rng, args...)` for `f ∈ (rand, randn)`

PlotsBase/test/runtests.jl

@@ -54,17 +54,17 @@ is_ci() || @eval using Gtk  # see JuliaPlots/VisualRegressionTests.jl/issues/30
 
 ref_name(i) = "ref" * lpad(i, 3, '0')
 
-const blacklist = if VERSION.major == 1 && VERSION.minor ≥ 9
+const broken_examples = if VERSION.major == 1 && VERSION.minor ≥ 9
     [
-        25,
+        25, # FIXME: remove, when StatsPlots supports Plots v2
         30, # FIXME: remove, when StatsPlots supports Plots v2
-        41,
-    ]  # FIXME: github.com/JuliaLang/julia/issues/47261
+        41, # FIXME: https://github.com/JuliaLang/julia/issues/47261
+    ]
 else
     []
 end
 if Sys.isapple()
-    push!(blacklist, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
+    push!(broken_examples, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
 end
 
 for name ∈ (

PlotsBase/test/test_backends.jl

@@ -66,7 +66,7 @@ is_pkgeval() || @testset "Backends" begin
         @test filesize(fn) > 1_000
     end
     Sys.islinux() && for be ∈ TEST_BACKENDS
-        skip = vcat(PlotsBase._backend_skips[be], blacklist)
+        skip = vcat(PlotsBase._backend_skips[be], broken_examples)
         PlotsBase.test_examples(be; skip, callback, disp = is_ci(), strict = true)  # `ci` display for coverage
         closeall()
     end

PlotsBase/test/test_reference.jl

@@ -105,14 +105,19 @@ end
 function image_comparison_facts(
     pkg::Symbol;
     skip = [],          # skip these examples (int index)
+    broken = [],        # known broken examples (int index)
     only = nothing,     # limit to these examples (int index)
     debug = false,      # print debug information ?
     sigma = [1, 1],     # number of pixels to "blur"
     tol = 1e-2,         # acceptable error (percent)
 )
     for i ∈ setdiff(1:length(PlotsBase._examples), skip)
         if only ≡ nothing || i in only
-            @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            if i ∈ broken
+                @test_broken success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            else
+                @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            end
         end
     end
 end
@@ -141,7 +146,8 @@ end
         image_comparison_facts(
             :gr,
             tol = PLOTS_IMG_TOL,
-            skip = vcat(PlotsBase._backend_skips[:gr], blacklist),
+            skip = vcat(PlotsBase._backend_skips[:gr]),
+            broken = broken_examples,
         )
     end
 end