diff options
Diffstat (limited to 'src/main/scala/com/iinteractive/test/harness')
7 files changed, 420 insertions, 0 deletions
diff --git a/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala new file mode 100644 index 0000000..56f32fd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala @@ -0,0 +1,16 @@ +package com.iinteractive.test.harness + +/** Classes that implement `MultiTestReporter` are capable of running a group + * of test classes, given their names. This typically involves some sort of + * summarization. + * + * @see [[com.iinteractive.test.harness.Reporter Reporter]]. + */ +trait MultiTestReporter { + /** Runs the test classes identifed by the list of fully qualified class + * names `testNames`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testNames: Seq[String]): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/Reporter.scala b/src/main/scala/com/iinteractive/test/harness/Reporter.scala new file mode 100644 index 0000000..a47444b --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/Reporter.scala @@ -0,0 +1,15 @@ +package com.iinteractive.test.harness + +/** Classes that implement `Reporter` are capable of running a test class, + * given its name. + * + * @see [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]]. + */ +trait Reporter { + /** Runs the test class identifed by the fully qualified class name + * `testName`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testName: String): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala new file mode 100644 index 0000000..cd555dd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala @@ -0,0 +1,42 @@ +package com.iinteractive.test.harness + +import java.io.{PipedInputStream,PipedOutputStream} +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +import com.iinteractive.test.tap.{Parser,TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.Test + +/** This is a trait for classes that run tests and summarize the results. It + * provides a single `runOneTest` method, which runs a test class and + * produces a stream of [[com.iinteractive.test.tap.TAPEvent TAP events]] + * which can be used to produce whatever summarized output you need. + */ +trait SummarizedTests { + /** Runs a single [[com.iinteractive.test.Test test]] instance, calling `cb` + * with each [[com.iinteractive.test.tap.TAPEvent TAP event]] as it is + * produced. + * + * @return The overall result of the test instance. + */ + protected def runOneTest (test: Test, cb: TAPEvent => Unit): TAPResult = { + val out = new PipedOutputStream + val in = new PipedInputStream(out) + + val testFuture = Future { + Console.withOut(out) { + test.runInHarness + } + out.close + } + + val parser = new Parser(cb) + val result = parser.parse(in) + in.close + Await.ready(testFuture, Duration.Inf) + + result + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala new file mode 100644 index 0000000..a5fe1e0 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala @@ -0,0 +1,192 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap.{TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.tap.{StartEvent,ResultEvent,PlanEvent,EndEvent} +import com.iinteractive.test.Test + +/** Runs a series of tests. The TAP output from these tests is parsed, and + * output is produced which is similar in style to Perl's + * [[https://metacpan.org/module/Test::Harness Test::Harness]]. + */ +class SummaryReporter extends MultiTestReporter with SummarizedTests { + def run (testNames: Seq[String]): Int = { + val results = runTests(testNames) + val success = results.values.forall(_.success) + printTestSummary(success, results) + if (success) 0 else 1 + } + + protected def runTests (testNames: Seq[String]): Map[String, TAPResult] = { + val maxLength = testNames.map(_.length).max + + testNames.map { name => + val callbackGenerator: () => TAPEvent => Unit = () => { + var width = 0 + var tests = 0 + var plan: Option[Int] = None + + def status = { + tests + "/" + plan.getOrElse("?") + } + + def printStatus (st: String) { + print("\r" + (" " * width) + "\r") + val line = + name + " " + ("." * (maxLength - name.length)) + ".. " + st + width = line.length + print(line) + Console.out.flush + } + + (e: TAPEvent) => e match { + case StartEvent => { + printStatus("") + } + case PlanEvent(p) => { + plan = Some(p.plan) + printStatus(status) + } + case ResultEvent(r) => { + tests += 1 + printStatus(status) + } + case EndEvent(result) => { + if (result.success) { + printStatus("") + println("ok") + } + else { + val results = result.results.length + val failed = result.results.count { t => + !t.passed && !t.directive.isDefined + } + + printStatus("") + println("Dubious, test returned " + result.exitCode) + println("Failed " + failed + "/" + results + " subtests") + } + } + case _ => () + } + } + + name -> runOneTest(newInstance[Test](name), callbackGenerator()) + }.toMap + } + + protected def printTestSummary ( + success: Boolean, + results: Map[String, TAPResult] + ) { + printSuccess(success) + printLongSummary(results) + printShortSummary(results) + printPassFail(success, results) + } + + private def printSuccess (success: Boolean) { + if (success) { + println("All tests successful.") + } + } + + private def printShortSummary (results: Map[String, TAPResult]) { + val files = results.size + val tests = results.values.map(_.results.length).sum + println("Files=" + files + ", Tests=" + tests) + } + + private def printLongSummary (results: Map[String, TAPResult]) { + val todoSucceeded = results.mapValues { r => + r.results.filter { t => + t.directive match { + case Some(TodoDirective(_)) => t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testsFailed = results.mapValues { r => + r.results.filter { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testNames = (todoSucceeded ++ testsFailed).keys + + if (testNames.nonEmpty) { + println("") + println("Test Summary Report") + println("-------------------") + + val maxLength = testNames.map(_.length).max + + for (name <- testNames) { + val result = results(name) + + println( + name + (" " * (maxLength - name.length)) + " " + + "(Tests: " + result.results.length + " " + + "Failed: " + testsFailed.getOrElse(name, Nil).length + ")" + ) + + if (testsFailed.isDefinedAt(name)) { + val fails = testsFailed(name) + println( + " Failed test" + (if (fails.length > 1) "s" else "") + ": " + + fails.map(_.number).mkString(", ") + ) + } + + if (todoSucceeded.isDefinedAt(name)) { + val todos = todoSucceeded(name) + println( + " TODO passed: " + + todos.map(_.number).mkString(", ") + ) + } + + val exitCode = results(name).exitCode + if (exitCode != 0) { + println(" Non-zero exit status: " + exitCode) + } + } + } + } + + private def printPassFail ( + success: Boolean, + results: Map[String, TAPResult] + ) { + if (success) { + println("Result: PASS") + } + else { + println("Result: FAIL") + + val testResults = results.values + + val testsFailed = testResults.map { r => + r.results.count { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_ > 0) + val failedFiles = testsFailed.size + val failedTests = testsFailed.sum + + val allFiles = testResults.size + val allTests = testResults.map(_.results.length).sum + + println( + "Failed " + failedFiles + "/" + allFiles + " test programs. " + + failedTests + "/" + allTests + " subtests failed." + ) + } + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala new file mode 100644 index 0000000..8a4dc9f --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala @@ -0,0 +1,12 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap +import com.iinteractive.test.Test + +/** Runs a single test. The TAP stream from that test is written directly to + * stdout/stderr. + */ +class TAPReporter extends Reporter { + def run (testName: String): Int = + newInstance[Test](testName).run +} diff --git a/src/main/scala/com/iinteractive/test/harness/TestHarness.scala b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala new file mode 100644 index 0000000..7b8f4ae --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala @@ -0,0 +1,110 @@ +package com.iinteractive.test.harness + +/** This is the entry point to running tests written with this library from + * the command line. Note that this library also implements the + * [[https://github.com/harrah/test-interface common testing interface]] for + * test libraries, so tests should also just work with `sbt test`. + * + * If this application is run and given just a single test class name, it + * will run that test and write its TAP stream to the console. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest + * ok 1 + * ok 2 + * 1..2 + * }}} + * + * If this application is run and given multiple test class names, it will + * run each of those tests, and present a summary report, similar to the one + * produces by + * [[https://metacpan.org/module/Test::Harness Perl's Test::Harness]]. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest1 MyTest2 + * MyTest1 .. ok + * MyTest2 .. ok + * All tests successful. + * Files=2, Tests=4 + * Result: PASS + * }}} + * + * This application also accepts a few command line options to customize its + * behavior: + * + * - `-r`: Alternative [[com.iinteractive.test.harness.Reporter Reporter]] + * class to use for running a single test. + * - `-R`: Alternative + * [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]] + * class to use for running a group of tests. Also enables using the + * MultiTestReporter for a single test. + * - `--help`: Prints usage information. + */ +object TestHarness { + import com.iinteractive.test.Test + + /** Entry point for the harness application. */ + def main (args: Array[String]) { + val opts = parseOpts(args.toList) + val single = opts("prefer-single").asInstanceOf[Boolean] + + val exitCode = if (single) { + val reporterName = opts("single-reporter").asInstanceOf[String] + val testName = opts("test-classes").asInstanceOf[List[String]].apply(0) + val reporter = newInstance[Reporter](reporterName) + reporter.run(testName) + } + else { + val reporterName = opts("multi-reporter").asInstanceOf[String] + val testNames = opts("test-classes").asInstanceOf[List[String]] + val reporter = newInstance[MultiTestReporter](reporterName) + reporter.run(testNames) + } + + sys.exit(exitCode) + } + + protected def parseOpts (args: List[String]): Map[String, Any] = args match { + case Nil => Map( + "single-reporter" -> "com.iinteractive.test.harness.TAPReporter", + "multi-reporter" -> "com.iinteractive.test.harness.SummaryReporter", + "prefer-single" -> true, + "test-classes" -> Nil + ) + + case "-r" :: singleReporter :: rest => + parseOpts(rest) + ("single-reporter" -> singleReporter) + + case "-R" :: multiReporter :: rest => + parseOpts(rest) ++ Map( + "multi-reporter" -> multiReporter, + "prefer-single" -> false + ) + + case "--help" :: rest => + usage(0) + + case `unknownOption` :: rest => + usage(1) + + case testClass :: rest => { + val opts = parseOpts(rest) + val tests = opts("test-classes").asInstanceOf[List[String]] + opts ++ Map( + "test-classes" -> (testClass :: tests), + "prefer-single" -> tests.isEmpty + ) + } + } + + protected def usage (exitCode: Int) = { + val out = if (exitCode == 0) Console.out else Console.err + out.println("harness [-r <single-reporter-class>]\n" + + " [-R <multi-reporter-class>]\n" + + " [--help]\n" + + " <test-class> [<test-class>...]\n") + sys.exit(exitCode) + } + + private val unknownOption = """^-.*""".r +} diff --git a/src/main/scala/com/iinteractive/test/harness/package.scala b/src/main/scala/com/iinteractive/test/harness/package.scala new file mode 100644 index 0000000..1b74f9d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/package.scala @@ -0,0 +1,33 @@ +package com.iinteractive.test + +/** Classes to handle running test instances and providing output. */ +package object harness { + import scala.reflect.{ClassTag,classTag} + + /** Loads `className`, returning the + * [[http://docs.oracle.com/javase/7/docs/api/java/lang/Class.html Class]] + * instance. + */ + def loadClass[T: ClassTag] (className: String): Class[_] = + classTag[T].runtimeClass.getClassLoader.loadClass(className) + + /** Loads `className` and creates a new instance of it, using the + * no-argument constructor. + */ + def newInstance[T: ClassTag] (className: String): T = + loadClass[T](className).newInstance.asInstanceOf[T] + + /** Loads `className` and creates a new instance of it, using a + * one-argument constructor. Passes `arg` as the argument to the + * constructor. + */ + def newInstance[T: ClassTag, U <: AnyRef: ClassTag] ( + className: String, + arg: U + ): T = { + val classObj = loadClass[T](className) + val argClassObj = classTag[U].runtimeClass + val constructor = classObj.getConstructor(argClassObj) + constructor.newInstance(arg).asInstanceOf[T] + } +} |