enable regex-filtering of testsets based on their descriptions

base/util.jl

@@ -799,25 +799,48 @@ end
 
 """
     Base.runtests(tests=["all"]; ncores=ceil(Int, Sys.CPU_THREADS / 2),
-                  exit_on_error=false, [seed])
+                  exit_on_error=false, [seed], [regex::AbstractString], [depths])
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
 strings, using `ncores` processors. If `exit_on_error` is `false`, when one test
 fails, all remaining tests in other files will still be run; they are otherwise discarded,
 when `exit_on_error == true`.
 If a seed is provided via the keyword argument, it is used to seed the
 global RNG in the context where the tests are run; otherwise the seed is chosen randomly.
+If a `regex` string is provided, it is used to create a `Regex` object and to filter
+testsets which are run based on their description string
+(i.e. `@testset "description" ...` is run if and only if `occursin(Regex(regex), "description")`).
+By default, only top-level testsets are filtered.
+If `depths` is provided, as an integer or as a collection thereof, the regex filtering occurs
+only for testsets whose nesting depth matches `depths` (top-level is 1).
 """
 function runtests(tests = ["all"]; ncores = ceil(Int, Sys.CPU_THREADS / 2),
-                  exit_on_error=false,
-                  seed::Union{BitInteger,Nothing}=nothing)
+                  exit_on_error::Bool=false,
+                  seed::Union{BitInteger,Nothing}=nothing,
+                  regex::Union{Nothing,AbstractString}=nothing,
+                  depths=nothing)
     if isa(tests,AbstractString)
         tests = split(tests)
     end
     exit_on_error && push!(tests, "--exit-on-error")
     seed !== nothing && push!(tests, "--seed=0x$(string(seed % UInt128, base=16))") # cast to UInt128 to avoid a minus sign
     ENV2 = copy(ENV)
     ENV2["JULIA_CPU_THREADS"] = "$ncores"
+    if regex !== nothing
+        ENV2["JULIA_TESTSET_REGEX"] = regex
+    end
+    ENV2["JULIA_TESTSET_DEPTHS"] = if depths === nothing
+        # by default, we want to apply a filtering regex at level 2, as all test files are implicitly wrapped in a @testset
+        get(ENV2, "JULIA_TESTSET_DEPTHS", "2")
+    else
+        if depths isa Integer
+            depths = [depths]
+        end
+        depths = Int.(depths)
+        any(<=(0), depths) && throw(ArgumentError("depths must be >= 1"))
+        join(string.(depths .+ 1), ',') # +1 to account for implicit level 1, see comment above
+    end
+
     try
         run(setenv(`$(julia_cmd()) $(joinpath(Sys.BINDIR::String,
             Base.DATAROOTDIR, "julia", "test", "runtests.jl")) $tests`, ENV2))

stdlib/Test/src/Test.jl

@@ -1095,26 +1095,33 @@ function testset_beginend(args, tests, source)
         # this empty loop is here to force the block to be compiled,
         # which is needed for backtrace scrubbing to work correctly.
         while false; end
-        push_testset(ts)
-        # we reproduce the logic of guardseed, but this function
-        # cannot be used as it changes slightly the semantic of @testset,
-        # by wrapping the body in a function
-        local RNG = default_rng()
-        oldrng = copy(RNG)
-        try
-            # RNG is re-seeded with its own seed to ease reproduce a failed test
-            Random.seed!(RNG.seed)
-            $(esc(tests))
-        catch err
-            err isa InterruptException && rethrow()
-            # something in the test block threw an error. Count that as an
-            # error in this test set
-            record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source))))
-        finally
-            copy!(RNG, oldrng)
+
+        rx = get(ENV, "JULIA_TESTSET_REGEX", nothing)
+        depths = get(ENV, "JULIA_TESTSET_DEPTHS", "1")
+        depths = parse.(Int, split(depths, ',')) .- 1
+
+        if get_testset_depth() ∉ depths || rx === nothing || occursin(Regex(rx), $desc)
+            push_testset(ts)
+            # we reproduce the logic of guardseed, but this function
+            # cannot be used as it changes slightly the semantic of @testset,
+            # by wrapping the body in a function
+            local RNG = default_rng()
+            oldrng = copy(RNG)
+            try
+                # RNG is re-seeded with its own seed to ease reproduce a failed test
+                Random.seed!(RNG.seed)
+                $(esc(tests))
+            catch err
+                err isa InterruptException && rethrow()
+                # something in the test block threw an error. Count that as an
+                # error in this test set
+                record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source))))
+            finally
+                copy!(RNG, oldrng)
+            end
+            pop_testset()
+            finish(ts)
         end
-        pop_testset()
-        finish(ts)
     end
     # preserve outer location if possible
     if tests isa Expr && tests.head === :block && !isempty(tests.args) && tests.args[1] isa LineNumberNode
@@ -1166,23 +1173,24 @@ function testset_forloop(args, testloop, source)
         _check_testset($testsettype, $(QuoteNode(testsettype.args[1])))
         # Trick to handle `break` and `continue` in the test code before
         # they can be handled properly by `finally` lowering.
-        if !first_iteration
-            pop_testset()
-            push!(arr, finish(ts))
-            # it's 1000 times faster to copy from tmprng rather than calling Random.seed!
-            copy!(RNG, tmprng)
-
-        end
-        ts = $(testsettype)($desc; $options...)
-        push_testset(ts)
-        first_iteration = false
-        try
-            $(esc(tests))
-        catch err
-            err isa InterruptException && rethrow()
-            # Something in the test block threw an error. Count that as an
-            # error in this test set
-            record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source))))
+        if rx === nothing || occursin(rx, $desc)
+            if !first_iteration
+                pop_testset()
+                push!(arr, finish(ts))
+                # it's 1000 times faster to copy from tmprng rather than calling Random.seed!
+                copy!(RNG, tmprng)
+            end
+            ts = $(testsettype)($desc; $options...)
+            push_testset(ts)
+            first_iteration = false
+            try
+                $(esc(tests))
+            catch err
+                err isa InterruptException && rethrow()
+                # Something in the test block threw an error. Count that as an
+                # error in this test set
+                record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source))))
+            end
         end
     end
     quote
@@ -1193,6 +1201,15 @@ function testset_forloop(args, testloop, source)
         local oldrng = copy(RNG)
         Random.seed!(RNG.seed)
         local tmprng = copy(RNG)
+        local rx = get(ENV, "JULIA_TESTSET_REGEX", nothing)
+        local depths = get(ENV, "JULIA_TESTSET_DEPTHS", "1")
+        if get_testset_depth()+1 ∉ parse.(Int, split(depths, ','))
+            rx = nothing
+        end
+        if rx !== nothing
+            rx = Regex(rx)
+        end
+
         try
             $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
         finally