From 02bde45b4db9f454978cf4021fc8fbe4b509d98a Mon Sep 17 00:00:00 2001 From: sanjiv sahayam Date: Tue, 20 Sep 2016 00:02:28 +1000 Subject: [PATCH] filters properties based on a criteria Allows properties to be filtered based on a supplied criteria. This allows running of a subtest of property tests for a given specification. The test criteria is specified by the '-f' option. Eg: Given the following properties for a String specification: 1. StringProps.String map 2. StringProps.String reverse when supplied with a "map" criteria, Scalacheck will only run the map property. 1. StringProps.String map On the commandline: scala -cp scalacheck.jar:. StringProps -f "map" Through SBT: test-only *StringProps -- -f map The primary motivation for creating this functionality was so that I could selectively run a subset of properties that were expensive to run every time. It also helps focus on the current property being tested without having to run all properties of a specification each time. I'm not sure I have covered all the bases here - I have tested this through SBT and the commandline, but I wonder if there are any other scenarios I should test this through. I'm happy to get some feedback on this and improve it as necessary. --- .gitignore | 4 + .../org/scalacheck/ScalaCheckFramework.scala | 84 ++++++++++--------- src/main/scala/org/scalacheck/Test.scala | 55 +++++++++--- .../org/scalacheck/util/CmdLineParser.scala | 2 + 4 files changed, 93 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 81dac8842..b941b9c3c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ target /examples/simple-sbt/project/ /examples/simple-sbt/target/ + +# ide +*.ctags_srcs/ +.tags diff --git a/src/main/scala/org/scalacheck/ScalaCheckFramework.scala b/src/main/scala/org/scalacheck/ScalaCheckFramework.scala index c5ecbbc84..dd80f2b59 100644 --- a/src/main/scala/org/scalacheck/ScalaCheckFramework.scala +++ b/src/main/scala/org/scalacheck/ScalaCheckFramework.scala @@ -14,6 +14,7 @@ import scala.language.reflectiveCalls import java.util.concurrent.atomic.AtomicInteger import org.scalacheck.Test.Parameters +import org.scalacheck.Test.matchRunFilter private abstract class ScalaCheckRunner extends Runner { @@ -99,49 +100,53 @@ private abstract class ScalaCheckRunner extends Runner { for ((`name`, prop) <- props) { val params = applyCmdParams(properties.foldLeft(Parameters.default)((params, props) => props.overrideParameters(params))) - val result = Test.check(params, prop) - - val event = new Event { - val status = result.status match { - case Test.Passed => Status.Success - case _:Test.Proved => Status.Success - case _:Test.Failed => Status.Failure - case Test.Exhausted => Status.Failure - case _:Test.PropException => Status.Error + val filter = params.runFilter + + if (filter.isEmpty || filter.exists(matchRunFilter(name, _))) { + val result = Test.check(params, prop) + + val event = new Event { + val status = result.status match { + case Test.Passed => Status.Success + case _:Test.Proved => Status.Success + case _:Test.Failed => Status.Failure + case Test.Exhausted => Status.Failure + case _:Test.PropException => Status.Error + } + val throwable = result.status match { + case Test.PropException(_, e, _) => new OptionalThrowable(e) + case _:Test.Failed => new OptionalThrowable( + new Exception(pretty(result, Params(0))) + ) + case _ => new OptionalThrowable() + } + val fullyQualifiedName = taskDef.fullyQualifiedName + val selector = new TestSelector(name) + val fingerprint = taskDef.fingerprint + val duration = -1L } - val throwable = result.status match { - case Test.PropException(_, e, _) => new OptionalThrowable(e) - case _:Test.Failed => new OptionalThrowable( - new Exception(pretty(result, Params(0))) - ) - case _ => new OptionalThrowable() - } - val fullyQualifiedName = taskDef.fullyQualifiedName - val selector = new TestSelector(name) - val fingerprint = taskDef.fingerprint - val duration = -1L - } - handler.handle(event) + handler.handle(event) - event.status match { - case Status.Success => successCount.incrementAndGet() - case Status.Error => errorCount.incrementAndGet() - case Status.Skipped => errorCount.incrementAndGet() - case Status.Failure => failureCount.incrementAndGet() - case _ => failureCount.incrementAndGet() + event.status match { + case Status.Success => successCount.incrementAndGet() + case Status.Error => errorCount.incrementAndGet() + case Status.Skipped => errorCount.incrementAndGet() + case Status.Failure => failureCount.incrementAndGet() + case _ => failureCount.incrementAndGet() + } + testCount.incrementAndGet() + + // TODO Stack traces should be reported through event + val verbosityOpts = Set("-verbosity", "-v") + val verbosity = + args.grouped(2).filter(twos => verbosityOpts(twos.head)) + .toSeq.headOption.map(_.last).map(_.toInt).getOrElse(0) + val s = if (result.passed) "+" else "!" + val n = if (name.isEmpty) taskDef.fullyQualifiedName else name + val logMsg = s"$s $n: ${pretty(result, Params(verbosity))}" + log(loggers, result.passed, logMsg) } - testCount.incrementAndGet() - - // TODO Stack traces should be reported through event - val verbosityOpts = Set("-verbosity", "-v") - val verbosity = - args.grouped(2).filter(twos => verbosityOpts(twos.head)) - .toSeq.headOption.map(_.last).map(_.toInt).getOrElse(0) - val s = if (result.passed) "+" else "!" - val n = if (name.isEmpty) taskDef.fullyQualifiedName else name - val logMsg = s"$s $n: ${pretty(result, Params(verbosity))}" - log(loggers, result.passed, logMsg) } Array.empty[Task] @@ -207,7 +212,6 @@ final class ScalaCheckFramework extends Framework { val args = _args val remoteArgs = _remoteArgs val loader = _loader - val applyCmdParams = Test.cmdLineParser.parseParams(args)._1.andThen { p => p.withTestCallback(new Test.TestCallback {}) .withCustomClassLoader(Some(loader)) diff --git a/src/main/scala/org/scalacheck/Test.scala b/src/main/scala/org/scalacheck/Test.scala index eca74694a..6369af6df 100644 --- a/src/main/scala/org/scalacheck/Test.scala +++ b/src/main/scala/org/scalacheck/Test.scala @@ -87,6 +87,15 @@ object Test { customClassLoader = customClassLoader ) + /** A test predicate to filter tests against. */ + val runFilter: Option[String] + + /** Create a copy of this [[Test.Parameters]] instance with + * [[Test.Parameters.runFilter]] set to the specified value. */ + def withRunFilter(runFilter: Option[String]): Parameters = cp( + runFilter = runFilter + ) + // private since we can't guarantee binary compatibility for this one private case class cp( minSuccessfulTests: Int = minSuccessfulTests, @@ -95,7 +104,8 @@ object Test { workers: Int = workers, testCallback: TestCallback = testCallback, maxDiscardRatio: Float = maxDiscardRatio, - customClassLoader: Option[ClassLoader] = customClassLoader + customClassLoader: Option[ClassLoader] = customClassLoader, + runFilter: Option[String] = runFilter ) extends Parameters override def toString = s"Parameters${cp.toString.substring(2)}" @@ -120,6 +130,7 @@ object Test { val testCallback: TestCallback = new TestCallback {} val maxDiscardRatio: Float = 5 val customClassLoader: Option[ClassLoader] = None + val runFilter = None } /** Verbose console reporter test parameters instance. */ @@ -237,9 +248,16 @@ object Test { val help = "Verbosity level" } + object OptRunFilter extends OpStrOpt { + val default = Parameters.default.runFilter + val names = Set("runFilter", "f") + val help = "Filter to match tests against" + } + val opts = Set[Opt[_]]( OptMinSuccess, OptMaxDiscardRatio, OptMinSize, - OptMaxSize, OptWorkers, OptVerbosity + OptMaxSize, OptWorkers, OptVerbosity, + OptRunFilter ) def parseParams(args: Array[String]): (Parameters => Parameters, List[String]) = { @@ -250,6 +268,7 @@ object Test { .withMinSize(optMap(OptMinSize): Int) .withMaxSize(optMap(OptMaxSize): Int) .withWorkers(optMap(OptWorkers): Int) + .withRunFilter(optMap(OptRunFilter): Option[String]) .withTestCallback(ConsoleReporter(optMap(OptVerbosity)): TestCallback) (params, us) } @@ -273,7 +292,6 @@ object Test { * the test results. */ def check(params: Parameters, p: Prop): Result = { import params._ - assertParams(params) val iterations = math.ceil(minSuccessfulTests / (workers: Double)) @@ -326,18 +344,31 @@ object Test { timedRes } + def matchRunFilter(testName: String, fragment: String): Boolean = { + testName.split("\\.") match { + case Array(prefix, suffix) if (suffix.contains(fragment)) => true + case Array(testName) if (testName.contains(fragment)) => true + case _ => false + } + } + /** Check a set of properties. */ def checkProperties(prms: Parameters, ps: Properties): Seq[(String,Result)] = { val params = ps.overrideParameters(prms) - ps.properties.map { case (name,p) => - val testCallback = new TestCallback { - override def onPropEval(n: String, t: Int, s: Int, d: Int) = - params.testCallback.onPropEval(name,t,s,d) - override def onTestResult(n: String, r: Result) = - params.testCallback.onTestResult(name,r) - } - val res = check(params.withTestCallback(testCallback), p) - (name,res) + + ps.properties.filter { + case (name, _) => prms.runFilter.fold(true)(matchRunFilter(name, _)) + } map { + case (name, p) => + val testCallback = new TestCallback { + override def onPropEval(n: String, t: Int, s: Int, d: Int) = + params.testCallback.onPropEval(name,t,s,d) + override def onTestResult(n: String, r: Result) = + params.testCallback.onTestResult(name,r) + } + + val res = check(params.withTestCallback(testCallback), p) + (name,res) } } } diff --git a/src/main/scala/org/scalacheck/util/CmdLineParser.scala b/src/main/scala/org/scalacheck/util/CmdLineParser.scala index 0975e51b1..638b0596d 100644 --- a/src/main/scala/org/scalacheck/util/CmdLineParser.scala +++ b/src/main/scala/org/scalacheck/util/CmdLineParser.scala @@ -23,6 +23,7 @@ private[scalacheck] trait CmdLineParser { trait IntOpt extends Opt[Int] trait FloatOpt extends Opt[Float] trait StrOpt extends Opt[String] + trait OpStrOpt extends Opt[Option[String]] class OptMap(private val opts: Map[Opt[_],Any] = Map.empty) { def apply(flag: Flag): Boolean = opts.contains(flag) @@ -73,6 +74,7 @@ private[scalacheck] trait CmdLineParser { case Some(o: IntOpt) => getInt(a2).map(v => parse(as, om.set(o -> v), us)) case Some(o: FloatOpt) => getFloat(a2).map(v => parse(as, om.set(o -> v), us)) case Some(o: StrOpt) => getStr(a2).map(v => parse(as, om.set(o -> v), us)) + case Some(o: OpStrOpt) => getStr(a2).map(v => parse(as, om.set(o -> Option(v)), us)) case _ => None }).getOrElse(parse(a2::as, om, us :+ a1)) }