filters properties based on a criteria

Allows properties to be filtered based on a supplied criteria. This
allows running of a subtest of property tests for a given specification.
The test criteria is specified by the '-f' option.

Eg:

Given the following properties for a String specification:
1. StringProps.String map
2. StringProps.String reverse

when supplied with a "map" criteria, Scalacheck will only run the map property.

1. StringProps.String map

On the commandline:

scala -cp scalacheck.jar:. StringProps -f "map"

Through SBT:

test-only *StringProps -- -f map

The primary motivation for creating this functionality was so that
I could selectively run a subset of properties that were expensive
to run every time. It also helps focus on the current property being
tested without having to run all properties of a specification each
time.

I'm not sure I have covered all the bases here - I have tested this
through SBT and the commandline, but I wonder if there are any other
scenarios I should test this through.

I'm happy to get some feedback on this and improve it as necessary.

.gitignore

@@ -18,3 +18,7 @@ target
 
 /examples/simple-sbt/project/
 /examples/simple-sbt/target/
+
+# ide
+*.ctags_srcs/
+.tags

src/main/scala/org/scalacheck/ScalaCheckFramework.scala

@@ -14,6 +14,7 @@ import scala.language.reflectiveCalls
 import java.util.concurrent.atomic.AtomicInteger
 
 import org.scalacheck.Test.Parameters
+import org.scalacheck.Test.matchRunFilter
 
 private abstract class ScalaCheckRunner extends Runner {
 
@@ -99,49 +100,53 @@ private abstract class ScalaCheckRunner extends Runner {
 
         for ((`name`, prop) <- props) {
           val params = applyCmdParams(properties.foldLeft(Parameters.default)((params, props) => props.overrideParameters(params)))
-          val result = Test.check(params, prop)
-
-          val event = new Event {
-            val status = result.status match {
-              case Test.Passed => Status.Success
-              case _:Test.Proved => Status.Success
-              case _:Test.Failed => Status.Failure
-              case Test.Exhausted => Status.Failure
-              case _:Test.PropException => Status.Error
+          val filter = params.runFilter
+
+          if (filter.isEmpty || filter.exists(matchRunFilter(name, _))) {
+            val result = Test.check(params, prop)
+
+            val event = new Event {
+              val status = result.status match {
+                case Test.Passed => Status.Success
+                case _:Test.Proved => Status.Success
+                case _:Test.Failed => Status.Failure
+                case Test.Exhausted => Status.Failure
+                case _:Test.PropException => Status.Error
+              }
+              val throwable = result.status match {
+                case Test.PropException(_, e, _) => new OptionalThrowable(e)
+                case _:Test.Failed => new OptionalThrowable(
+                  new Exception(pretty(result, Params(0)))
+                )
+                case _ => new OptionalThrowable()
+              }
+              val fullyQualifiedName = taskDef.fullyQualifiedName
+              val selector = new TestSelector(name)
+              val fingerprint = taskDef.fingerprint
+              val duration = -1L
             }
-            val throwable = result.status match {
-              case Test.PropException(_, e, _) => new OptionalThrowable(e)
-              case _:Test.Failed => new OptionalThrowable(
-                new Exception(pretty(result, Params(0)))
-              )
-              case _ => new OptionalThrowable()
-            }
-            val fullyQualifiedName = taskDef.fullyQualifiedName
-            val selector = new TestSelector(name)
-            val fingerprint = taskDef.fingerprint
-            val duration = -1L
-          }
 
-          handler.handle(event)
+            handler.handle(event)
 
-          event.status match {
-            case Status.Success => successCount.incrementAndGet()
-            case Status.Error => errorCount.incrementAndGet()
-            case Status.Skipped => errorCount.incrementAndGet()
-            case Status.Failure => failureCount.incrementAndGet()
-            case _ => failureCount.incrementAndGet()
+            event.status match {
+              case Status.Success => successCount.incrementAndGet()
+              case Status.Error => errorCount.incrementAndGet()
+              case Status.Skipped => errorCount.incrementAndGet()
+              case Status.Failure => failureCount.incrementAndGet()
+              case _ => failureCount.incrementAndGet()
+            }
+            testCount.incrementAndGet()
+
+            // TODO Stack traces should be reported through event
+            val verbosityOpts = Set("-verbosity", "-v")
+            val verbosity =
+              args.grouped(2).filter(twos => verbosityOpts(twos.head))
+              .toSeq.headOption.map(_.last).map(_.toInt).getOrElse(0)
+            val s = if (result.passed) "+" else "!"
+            val n = if (name.isEmpty) taskDef.fullyQualifiedName else name
+            val logMsg = s"$s $n: ${pretty(result, Params(verbosity))}"
+            log(loggers, result.passed, logMsg)
           }
-          testCount.incrementAndGet()
-
-          // TODO Stack traces should be reported through event
-          val verbosityOpts = Set("-verbosity", "-v")
-          val verbosity =
-            args.grouped(2).filter(twos => verbosityOpts(twos.head))
-            .toSeq.headOption.map(_.last).map(_.toInt).getOrElse(0)
-          val s = if (result.passed) "+" else "!"
-          val n = if (name.isEmpty) taskDef.fullyQualifiedName else name
-          val logMsg = s"$s $n: ${pretty(result, Params(verbosity))}"
-          log(loggers, result.passed, logMsg)
         }
 
         Array.empty[Task]
@@ -207,7 +212,6 @@ final class ScalaCheckFramework extends Framework {
     val args = _args
     val remoteArgs = _remoteArgs
     val loader = _loader
-
     val applyCmdParams = Test.cmdLineParser.parseParams(args)._1.andThen {
       p => p.withTestCallback(new Test.TestCallback {})
           .withCustomClassLoader(Some(loader))

src/main/scala/org/scalacheck/Test.scala

@@ -87,6 +87,15 @@ object Test {
       customClassLoader = customClassLoader
     )
 
+    /** A test predicate to filter tests against. */
+    val runFilter: Option[String]
+
+    /** Create a copy of this [[Test.Parameters]] instance with
+     *  [[Test.Parameters.runFilter]] set to the specified value. */
+    def withRunFilter(runFilter: Option[String]): Parameters = cp(
+      runFilter = runFilter
+    )
+
     // private since we can't guarantee binary compatibility for this one
     private case class cp(
       minSuccessfulTests: Int = minSuccessfulTests,
@@ -95,7 +104,8 @@ object Test {
       workers: Int = workers,
       testCallback: TestCallback = testCallback,
       maxDiscardRatio: Float = maxDiscardRatio,
-      customClassLoader: Option[ClassLoader] = customClassLoader
+      customClassLoader: Option[ClassLoader] = customClassLoader,
+      runFilter: Option[String] = runFilter
     ) extends Parameters
 
     override def toString = s"Parameters${cp.toString.substring(2)}"
@@ -120,6 +130,7 @@ object Test {
       val testCallback: TestCallback = new TestCallback {}
       val maxDiscardRatio: Float = 5
       val customClassLoader: Option[ClassLoader] = None
+      val runFilter = None
     }
 
     /** Verbose console reporter test parameters instance. */
@@ -237,9 +248,16 @@ object Test {
       val help = "Verbosity level"
     }
 
+    object OptRunFilter extends OpStrOpt {
+      val default = Parameters.default.runFilter
+      val names = Set("runFilter", "f")
+      val help = "Filter to match tests against"
+    }
+
     val opts = Set[Opt[_]](
       OptMinSuccess, OptMaxDiscardRatio, OptMinSize,
-      OptMaxSize, OptWorkers, OptVerbosity
+      OptMaxSize, OptWorkers, OptVerbosity,
+      OptRunFilter
     )
 
     def parseParams(args: Array[String]): (Parameters => Parameters, List[String]) = {
@@ -250,6 +268,7 @@ object Test {
         .withMinSize(optMap(OptMinSize): Int)
         .withMaxSize(optMap(OptMaxSize): Int)
         .withWorkers(optMap(OptWorkers): Int)
+        .withRunFilter(optMap(OptRunFilter): Option[String])
         .withTestCallback(ConsoleReporter(optMap(OptVerbosity)): TestCallback)
       (params, us)
     }
@@ -273,7 +292,6 @@ object Test {
    *  the test results. */
   def check(params: Parameters, p: Prop): Result = {
     import params._
-
     assertParams(params)
 
     val iterations = math.ceil(minSuccessfulTests / (workers: Double))
@@ -326,18 +344,31 @@ object Test {
     timedRes
   }
 
+  def matchRunFilter(testName: String, fragment: String): Boolean = {
+    testName.split("\\.") match {
+      case Array(prefix, suffix) if (suffix.contains(fragment)) => true
+      case Array(testName) if (testName.contains(fragment)) => true
+      case _ => false
+    }
+  }
+
   /** Check a set of properties. */
   def checkProperties(prms: Parameters, ps: Properties): Seq[(String,Result)] = {
     val params = ps.overrideParameters(prms)
-    ps.properties.map { case (name,p) =>
-      val testCallback = new TestCallback {
-        override def onPropEval(n: String, t: Int, s: Int, d: Int) =
-          params.testCallback.onPropEval(name,t,s,d)
-        override def onTestResult(n: String, r: Result) =
-          params.testCallback.onTestResult(name,r)
-      }
-      val res = check(params.withTestCallback(testCallback), p)
-      (name,res)
+
+    ps.properties.filter {
+      case (name, _) => prms.runFilter.fold(true)(matchRunFilter(name, _))
+    } map {
+      case (name, p)  =>
+        val testCallback = new TestCallback {
+          override def onPropEval(n: String, t: Int, s: Int, d: Int) =
+            params.testCallback.onPropEval(name,t,s,d)
+          override def onTestResult(n: String, r: Result) =
+            params.testCallback.onTestResult(name,r)
+        }
+
+        val res = check(params.withTestCallback(testCallback), p)
+        (name,res)
     }
   }
 }

src/main/scala/org/scalacheck/util/CmdLineParser.scala

@@ -23,6 +23,7 @@ private[scalacheck] trait CmdLineParser {
   trait IntOpt extends Opt[Int]
   trait FloatOpt extends Opt[Float]
   trait StrOpt extends Opt[String]
+  trait OpStrOpt extends Opt[Option[String]]
 
   class OptMap(private val opts: Map[Opt[_],Any] = Map.empty) {
     def apply(flag: Flag): Boolean = opts.contains(flag)
@@ -73,6 +74,7 @@ private[scalacheck] trait CmdLineParser {
         case Some(o: IntOpt) => getInt(a2).map(v => parse(as, om.set(o -> v), us))
         case Some(o: FloatOpt) => getFloat(a2).map(v => parse(as, om.set(o -> v), us))
         case Some(o: StrOpt) => getStr(a2).map(v => parse(as, om.set(o -> v), us))
+        case Some(o: OpStrOpt) => getStr(a2).map(v => parse(as, om.set(o -> Option(v)), us))
         case _ => None
       }).getOrElse(parse(a2::as, om, us :+ a1))
     }