From a4f00bf58d04cf55b177231edd1f0840427b52b4 Mon Sep 17 00:00:00 2001
From: Etienne Kneuss <colder@php.net>
Date: Thu, 29 Aug 2013 16:34:31 +0200
Subject: [PATCH] Implement performance regression testing
Ensures that tests run in an acceptable amount of time.
Needs at least 5 runs. Can cause failures of test-only due to
warm-up and class loading effects.
---
.gitignore | 1 +
build.sbt | 6 +-
src/test/scala/leon/test/LeonTestSuite.scala | 86 +++++++++++++++++++
.../test/evaluators/EvaluatorsTests.scala | 4 +-
.../scala/leon/test/purescala/DataGen.scala | 2 +-
.../leon/test/purescala/LikelyEqSuite.scala | 7 +-
.../purescala/TreeNormalizationsTests.scala | 4 +-
.../leon/test/purescala/TreeOpsTests.scala | 4 +-
.../scala/leon/test/purescala/TreeTests.scala | 7 +-
.../test/solvers/TimeoutSolverTests.scala | 7 +-
.../test/solvers/z3/FairZ3SolverTests.scala | 4 +-
.../solvers/z3/FairZ3SolverTestsNewAPI.scala | 4 +-
.../z3/UninterpretedZ3SolverTests.scala | 4 +-
.../leon/test/synthesis/AlgebraSuite.scala | 7 +-
.../test/synthesis/LinearEquationsSuite.scala | 7 +-
.../leon/test/synthesis/SynthesisSuite.scala | 3 +-
.../PureScalaVerificationRegression.scala | 4 +-
.../XLangVerificationRegression.scala | 4 +-
18 files changed, 117 insertions(+), 48 deletions(-)
create mode 100644 src/test/scala/leon/test/LeonTestSuite.scala
diff --git a/.gitignore b/.gitignore
index 12c115fd5..eac146feb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@ target
/leon
/setupenv
/leon-bench
+/.test-history
# synthesis
derivation*.dot
diff --git a/build.sbt b/build.sbt
index 651417296..4fe1fed09 100644
--- a/build.sbt
+++ b/build.sbt
@@ -24,10 +24,14 @@ resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/release
libraryDependencies ++= Seq(
"org.scala-lang" % "scala-compiler" % "2.10.2",
- "org.scalatest" %% "scalatest" % "1.9.1" excludeAll(ExclusionRule(organization="org.scala-lang")),
+ "org.scalatest" % "scalatest_2.10" % "2.0.M5b" % "test" excludeAll(ExclusionRule(organization="org.scala-lang")),
"com.typesafe.akka" %% "akka-actor" % "2.2.0" excludeAll(ExclusionRule(organization="org.scala-lang"))
)
fork in run := true
fork in test := true
+
+logBuffered in Test := false
+
+testOptions in Test += Tests.Argument("-oD")
diff --git a/src/test/scala/leon/test/LeonTestSuite.scala b/src/test/scala/leon/test/LeonTestSuite.scala
new file mode 100644
index 000000000..2dd0a92eb
--- /dev/null
+++ b/src/test/scala/leon/test/LeonTestSuite.scala
@@ -0,0 +1,86 @@
+package leon.test
+import scala.io.Source
+import org.scalatest._
+
+trait LeonTestSuite extends FunSuite {
+ def now() = {
+ System.currentTimeMillis
+ }
+
+ case class Statistics(values: List[Long]) {
+ val n = values.size
+ val avg = values.sum.toDouble/n
+ val stddev = Math.sqrt(Math.abs(values.map(_.toDouble - avg).sum/n))
+
+ def accountsFor(ms: Long) = {
+ if (n < 5) {
+ true
+ } else {
+ val msd = ms.toDouble
+ (msd < avg + 3*stddev + 20)
+ }
+ }
+
+ def withValue(v: Long) = this.copy(v :: values)
+ }
+
+ def testIdentifier(name: String): String = {
+ (this.getClass.getName+"-"+name).replaceAll("[^0-9a-zA-Z-]", "")
+ }
+
+ def bookKeepingFile(id: String) = {
+ import java.io.File
+
+ val f = new File(System.getProperty("user.dir")+"/.test-history/"+id+".log");
+
+ f.getParentFile.mkdirs()
+
+ f
+ }
+
+ def getStats(id: String): Statistics = {
+ val f = bookKeepingFile(id)
+
+ if (f.canRead()) {
+ Statistics(Source.fromFile(f).getLines.flatMap{ line =>
+ val l = line.trim
+ if (l.length > 0) {
+ Some(line.toLong)
+ } else {
+ None
+ }
+ }.toList)
+ } else {
+ Statistics(Nil)
+ }
+ }
+
+ def storeStats(id: String, stats: Statistics) {
+ import java.io.FileWriter
+
+ val f = bookKeepingFile(id)
+
+ val fw = new FileWriter(f, true)
+ fw.write(stats.values.head+"\n")
+ fw.close
+ }
+
+ override def test(name: String, tags: Tag*)(body: => Unit) {
+ super.test(name, tags: _*) {
+ val id = testIdentifier(name)
+ val ts = now()
+
+ body
+
+ val total = now()-ts
+
+ val stats = getStats(id)
+
+ if (!stats.accountsFor(total)) {
+ fail("Test took too long to run: "+total+"ms (avg: "+stats.avg+", stddev: "+stats.stddev+")")
+ }
+
+ storeStats(id, stats.withValue(total))
+ }
+ }
+}
diff --git a/src/test/scala/leon/test/evaluators/EvaluatorsTests.scala b/src/test/scala/leon/test/evaluators/EvaluatorsTests.scala
index 9fd88e13b..edcc0c9b5 100644
--- a/src/test/scala/leon/test/evaluators/EvaluatorsTests.scala
+++ b/src/test/scala/leon/test/evaluators/EvaluatorsTests.scala
@@ -14,9 +14,7 @@ import leon.purescala.Definitions._
import leon.purescala.Trees._
import leon.purescala.TypeTrees._
-import org.scalatest.FunSuite
-
-class EvaluatorsTests extends FunSuite {
+class EvaluatorsTests extends LeonTestSuite {
private implicit lazy val leonContext = LeonContext(
settings = Settings(
synthesis = false,
diff --git a/src/test/scala/leon/test/purescala/DataGen.scala b/src/test/scala/leon/test/purescala/DataGen.scala
index b188291bb..99659fb69 100644
--- a/src/test/scala/leon/test/purescala/DataGen.scala
+++ b/src/test/scala/leon/test/purescala/DataGen.scala
@@ -16,7 +16,7 @@ import leon.evaluators._
import org.scalatest.FunSuite
-class DataGen extends FunSuite {
+class DataGen extends LeonTestSuite {
private implicit lazy val leonContext = LeonContext(
settings = Settings(
synthesis = false,
diff --git a/src/test/scala/leon/test/purescala/LikelyEqSuite.scala b/src/test/scala/leon/test/purescala/LikelyEqSuite.scala
index 3a4fa9371..54531aa35 100644
--- a/src/test/scala/leon/test/purescala/LikelyEqSuite.scala
+++ b/src/test/scala/leon/test/purescala/LikelyEqSuite.scala
@@ -1,14 +1,13 @@
/* Copyright 2009-2013 EPFL, Lausanne */
-package leon.test.purescala
-
-import org.scalatest.FunSuite
+package leon.test
+package purescala
import leon.purescala.Common._
import leon.purescala.Trees._
-class LikelyEqSuite extends FunSuite {
+class LikelyEqSuite extends LeonTestSuite {
def i(x: Int) = IntLiteral(x)
val xId = FreshIdentifier("x")
diff --git a/src/test/scala/leon/test/purescala/TreeNormalizationsTests.scala b/src/test/scala/leon/test/purescala/TreeNormalizationsTests.scala
index 42ac12758..5d788ef2e 100644
--- a/src/test/scala/leon/test/purescala/TreeNormalizationsTests.scala
+++ b/src/test/scala/leon/test/purescala/TreeNormalizationsTests.scala
@@ -10,9 +10,7 @@ import leon.purescala.Trees._
import leon.purescala.TreeOps._
import leon.purescala.TreeNormalizations._
-import org.scalatest.FunSuite
-
-class TreeNormalizationsTests extends FunSuite {
+class TreeNormalizationsTests extends LeonTestSuite {
def i(x: Int) = IntLiteral(x)
val xId = FreshIdentifier("x").setType(Int32Type)
diff --git a/src/test/scala/leon/test/purescala/TreeOpsTests.scala b/src/test/scala/leon/test/purescala/TreeOpsTests.scala
index 42ccca719..823383baf 100644
--- a/src/test/scala/leon/test/purescala/TreeOpsTests.scala
+++ b/src/test/scala/leon/test/purescala/TreeOpsTests.scala
@@ -11,9 +11,7 @@ import leon.purescala.Trees._
import leon.purescala.TypeTrees._
import leon.purescala.TreeOps._
-import org.scalatest.FunSuite
-
-class TreeOpsTests extends FunSuite {
+class TreeOpsTests extends LeonTestSuite {
private val silentContext = LeonContext(reporter = new TestSilentReporter)
test("Path-aware simplifications") {
diff --git a/src/test/scala/leon/test/purescala/TreeTests.scala b/src/test/scala/leon/test/purescala/TreeTests.scala
index 45c341d49..0d46e8fb9 100644
--- a/src/test/scala/leon/test/purescala/TreeTests.scala
+++ b/src/test/scala/leon/test/purescala/TreeTests.scala
@@ -1,15 +1,14 @@
/* Copyright 2009-2013 EPFL, Lausanne */
-package leon.test.purescala
+package leon.test
+package purescala
import leon.purescala.Common._
import leon.purescala.Definitions._
import leon.purescala.Trees._
import leon.purescala.TypeTrees._
-import org.scalatest.FunSuite
-
-class TreeTests extends FunSuite {
+class TreeTests extends LeonTestSuite {
test("And- and Or- simplifications") {
val x = Variable(FreshIdentifier("x").setType(BooleanType))
diff --git a/src/test/scala/leon/test/solvers/TimeoutSolverTests.scala b/src/test/scala/leon/test/solvers/TimeoutSolverTests.scala
index 462787790..954d30e65 100644
--- a/src/test/scala/leon/test/solvers/TimeoutSolverTests.scala
+++ b/src/test/scala/leon/test/solvers/TimeoutSolverTests.scala
@@ -1,8 +1,7 @@
/* Copyright 2009-2013 EPFL, Lausanne */
-package leon.test.solvers
-
-import org.scalatest.FunSuite
+package leon.test
+package solvers
import leon._
import leon.solvers._
@@ -11,7 +10,7 @@ import leon.purescala.Definitions._
import leon.purescala.Trees._
import leon.purescala.TypeTrees._
-class TimeoutSolverTests extends FunSuite {
+class TimeoutSolverTests extends LeonTestSuite {
private class IdioticSolver(ctx : LeonContext) extends Solver(ctx) with NaiveIncrementalSolver {
val name = "Idiotic"
val description = "Loops when it doesn't know"
diff --git a/src/test/scala/leon/test/solvers/z3/FairZ3SolverTests.scala b/src/test/scala/leon/test/solvers/z3/FairZ3SolverTests.scala
index 4e479a835..68a41d0ce 100644
--- a/src/test/scala/leon/test/solvers/z3/FairZ3SolverTests.scala
+++ b/src/test/scala/leon/test/solvers/z3/FairZ3SolverTests.scala
@@ -14,9 +14,7 @@ import leon.purescala.TypeTrees._
import leon.solvers.Solver
import leon.solvers.z3.FairZ3Solver
-import org.scalatest.FunSuite
-
-class FairZ3SolverTests extends FunSuite {
+class FairZ3SolverTests extends LeonTestSuite {
private var testCounter : Int = 0
private def solverCheck(solver : Solver, expr : Expr, expected : Option[Boolean], msg : String) = {
testCounter += 1
diff --git a/src/test/scala/leon/test/solvers/z3/FairZ3SolverTestsNewAPI.scala b/src/test/scala/leon/test/solvers/z3/FairZ3SolverTestsNewAPI.scala
index 53bac4d7c..802bd13f0 100644
--- a/src/test/scala/leon/test/solvers/z3/FairZ3SolverTestsNewAPI.scala
+++ b/src/test/scala/leon/test/solvers/z3/FairZ3SolverTestsNewAPI.scala
@@ -14,9 +14,7 @@ import leon.purescala.TypeTrees._
import leon.solvers.Solver
import leon.solvers.z3.FairZ3Solver
-import org.scalatest.FunSuite
-
-class FairZ3SolverTestsNewAPI extends FunSuite {
+class FairZ3SolverTestsNewAPI extends LeonTestSuite {
private var testCounter : Int = 0
private def solverCheck(solver : Solver, expr : Expr, expected : Option[Boolean], msg : String) = {
testCounter += 1
diff --git a/src/test/scala/leon/test/solvers/z3/UninterpretedZ3SolverTests.scala b/src/test/scala/leon/test/solvers/z3/UninterpretedZ3SolverTests.scala
index e9d87d641..339d5bf2a 100644
--- a/src/test/scala/leon/test/solvers/z3/UninterpretedZ3SolverTests.scala
+++ b/src/test/scala/leon/test/solvers/z3/UninterpretedZ3SolverTests.scala
@@ -14,9 +14,7 @@ import leon.purescala.TypeTrees._
import leon.solvers.Solver
import leon.solvers.z3.UninterpretedZ3Solver
-import org.scalatest.FunSuite
-
-class UninterpretedZ3SolverTests extends FunSuite {
+class UninterpretedZ3SolverTests extends LeonTestSuite {
private var testCounter : Int = 0
private def solverCheck(solver : Solver, expr : Expr, expected : Option[Boolean], msg : String) = {
testCounter += 1
diff --git a/src/test/scala/leon/test/synthesis/AlgebraSuite.scala b/src/test/scala/leon/test/synthesis/AlgebraSuite.scala
index 88fce1450..aef22779e 100644
--- a/src/test/scala/leon/test/synthesis/AlgebraSuite.scala
+++ b/src/test/scala/leon/test/synthesis/AlgebraSuite.scala
@@ -1,12 +1,11 @@
/* Copyright 2009-2013 EPFL, Lausanne */
-package leon.test.synthesis
-
-import org.scalatest.FunSuite
+package leon.test
+package synthesis
import leon.synthesis.Algebra._
-class AlgebraSuite extends FunSuite {
+class AlgebraSuite extends LeonTestSuite {
test("remainder") {
assert(remainder(1,1) === 0)
diff --git a/src/test/scala/leon/test/synthesis/LinearEquationsSuite.scala b/src/test/scala/leon/test/synthesis/LinearEquationsSuite.scala
index bd83fba3f..03b3e66a8 100644
--- a/src/test/scala/leon/test/synthesis/LinearEquationsSuite.scala
+++ b/src/test/scala/leon/test/synthesis/LinearEquationsSuite.scala
@@ -1,8 +1,7 @@
/* Copyright 2009-2013 EPFL, Lausanne */
-package leon.test.synthesis
-
-import org.scalatest.FunSuite
+package leon.test
+package synthesis
import leon.purescala.Trees._
import leon.purescala.TypeTrees._
@@ -12,7 +11,7 @@ import leon.test.purescala.LikelyEq
import leon.synthesis.LinearEquations._
-class LinearEquationsSuite extends FunSuite {
+class LinearEquationsSuite extends LeonTestSuite {
def i(x: Int) = IntLiteral(x)
diff --git a/src/test/scala/leon/test/synthesis/SynthesisSuite.scala b/src/test/scala/leon/test/synthesis/SynthesisSuite.scala
index 0f512690d..d134c980e 100644
--- a/src/test/scala/leon/test/synthesis/SynthesisSuite.scala
+++ b/src/test/scala/leon/test/synthesis/SynthesisSuite.scala
@@ -12,12 +12,11 @@ import leon.solvers.Solver
import leon.synthesis._
import leon.synthesis.utils._
-import org.scalatest.FunSuite
import org.scalatest.matchers.ShouldMatchers._
import java.io.{BufferedWriter, FileWriter, File}
-class SynthesisSuite extends FunSuite {
+class SynthesisSuite extends LeonTestSuite {
private var counter : Int = 0
private def nextInt() : Int = {
counter += 1
diff --git a/src/test/scala/leon/test/verification/PureScalaVerificationRegression.scala b/src/test/scala/leon/test/verification/PureScalaVerificationRegression.scala
index 5dd93cbed..45374ca5e 100644
--- a/src/test/scala/leon/test/verification/PureScalaVerificationRegression.scala
+++ b/src/test/scala/leon/test/verification/PureScalaVerificationRegression.scala
@@ -6,13 +6,11 @@ package verification
import leon.verification.{AnalysisPhase,VerificationReport}
-import org.scalatest.FunSuite
-
import java.io.File
import TestUtils._
-class PureScalaVerificationRegression extends FunSuite {
+class PureScalaVerificationRegression extends LeonTestSuite {
private var counter : Int = 0
private def nextInt() : Int = {
counter += 1
diff --git a/src/test/scala/leon/test/verification/XLangVerificationRegression.scala b/src/test/scala/leon/test/verification/XLangVerificationRegression.scala
index d85a5dd6f..64b523de9 100644
--- a/src/test/scala/leon/test/verification/XLangVerificationRegression.scala
+++ b/src/test/scala/leon/test/verification/XLangVerificationRegression.scala
@@ -6,13 +6,11 @@ package verification
import leon.verification.{AnalysisPhase,VerificationReport}
-import org.scalatest.FunSuite
-
import java.io.File
import TestUtils._
-class XLangVerificationRegression extends FunSuite {
+class XLangVerificationRegression extends LeonTestSuite {
private var counter : Int = 0
private def nextInt() : Int = {
counter += 1
--
GitLab