diff options
Diffstat (limited to 'src/main/scala/com/iinteractive/test')
23 files changed, 2042 insertions, 0 deletions
diff --git a/src/main/scala/com/iinteractive/test/ExternalTest.scala b/src/main/scala/com/iinteractive/test/ExternalTest.scala new file mode 100644 index 0000000..7855aa8 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/ExternalTest.scala @@ -0,0 +1,73 @@ +package com.iinteractive.test + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.concurrent.Future._ +import scala.annotation.tailrec + +/** Runs an external process which emits TAP, and parses it as a test. + * + * This test class can be used if you have existing tests that you would like + * to be able to include in a test suite using this framework. You just need + * to write a test class for each external test that looks like this: + * + * {{{ + * class MyTest1 extends ExternalTest("perl", "t/basic.t") + * }}} + * + * This will run your external process, and use its TAP stream as its output. + * This will allow it to, for instance, be a part of the test suite that runs + * via `sbt test`. As with any other test class, its stdout and stderr will + * be sent to `Console.out` and `Console.err`, where they can be overridden + * as needed. + */ +class ExternalTest (cmdLine: String*) extends Test { + protected def runTests (raw: Boolean): Int = { + val processBuilder = new ProcessBuilder(cmdLine: _*) + + // Ensure that if stdout and stderr are both pointing to the same place (a + // terminal or file or something) that they remain synchronized. This is + // only possible if the endpoint they are streaming to is the same place + // underneath, with no extra processing in between. (This is safe because + // stdout is typically line-buffered, and we only ever output a line at a + // time when writing TAP, so in theory, buffering of the underlying file + // descriptor shouldn't make a difference here) + if (Console.out eq System.out) { + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT) + } + if (Console.err eq System.err) { + processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT) + } + + val process = processBuilder.start + + val streams = Seq( + Console.out -> process.getInputStream, + Console.err -> process.getErrorStream + ) + + val listeners = streams.map { case (out, in) => + Future { + val buf = new Array[Byte](1024) + + @tailrec + def read { + val bytes = in.read(buf) + if (bytes >= 0) { + out.print(new String(buf.take(bytes))) + read + } + } + + read + true + } + } + + val exitCode = process.waitFor + Await.ready(Future.sequence(listeners), Duration.Inf) + exitCode + } +} diff --git a/src/main/scala/com/iinteractive/test/Test.scala b/src/main/scala/com/iinteractive/test/Test.scala new file mode 100644 index 0000000..2161de3 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/Test.scala @@ -0,0 +1,34 @@ +package com.iinteractive.test + +/** Base trait for test classes in this framework. Any tests that should be + * autodiscovered by `sbt test` should extend this trait, and implement + * [[runTests]]. + */ +trait Test { + /** Runs the test. The TAP stream will be written to Console.out and + * Console.err, so you can swap these out as required in order to parse it. + * + * @return The exit code that the test produced. Success is indicated by 0, + * failure to run the correct number of tests by 255, and any other + * failure by the number of tests that failed. This should be used + * by reporters which run a single test, which can call + * `sys.exit(exitCode)` + */ + def run: Int = + runTests(false) + + /** Runs the test just like [[run]], but in a way that makes sense when test + * results are being summarized rather than directly displayed. + * + * Summarizing test reporters tend to repeatedly update the same line on + * the terminal, so this method makes calls to + * [[com.iinteractive.test.tap.TestBuilder#diag diag]] (which sends + * messages to stderr, where they are typically displayed as-is) prefix the + * message with a newline, to ensure that the output starts on its own + * line. + */ + def runInHarness: Int = + runTests(true) + + protected def runTests (terminalInUse: Boolean): Int +} diff --git a/src/main/scala/com/iinteractive/test/TestMore.scala b/src/main/scala/com/iinteractive/test/TestMore.scala new file mode 100644 index 0000000..eb5210b --- /dev/null +++ b/src/main/scala/com/iinteractive/test/TestMore.scala @@ -0,0 +1,519 @@ +package com.iinteractive.test + +import scala.util.matching.Regex + +import com.iinteractive.test.tap.TestBuilder + +/** This class is an implementation of the excellent + * [[https://metacpan.org/module/Test::More Test::More]] testing library for + * Perl. It provides a simple assertion-based testing API, which produces + * [[http://en.wikipedia.org/wiki/Test_Anything_Protocol TAP]], which can be + * parsed by any TAP consumer. This library includes several TAP-consuming + * harnesses to use with tests using this class, including one that supports + * testing via `sbt test`. + * + * ==Basics== + * + * The most basic test looks like this: + * + * <pre> + * class MyTest extends TestMore { + * ok(true) + * } + * </pre> + * + * This runs a test containing a single assertion. This will generate a TAP + * stream that looks like this: + * + * <pre> + * ok 1 + * 1..1 + * </pre> + * + * which can be parsed by one of the test harnesses provided by this library. + * + * ==Running tests== + * + * The simplest way to run tests is through sbt. You can register this + * framework with sbt by adding this line to your `build.sbt` file: + * + * <pre> + * testFrameworks += new TestFramework("com.iinteractive.test.sbt.Framework") + * </pre> + * + * Then, any classes in your test directory which extend `TestMore` will be + * automatically detected and run. + * + * ==Assertions== + * + * This class contains many more assertion methods than just `ok`. Here is a + * more extensive example (borrowed from Test::More's documentation): + * + * <pre> + * class MyTest extends TestMore { + * ok(got == expected, testName) + * + * is(got, expected, testName) + * isnt(got, expected, testName) + * + * diag("here's what went wrong") + * + * like(got, """expected""".r, testName) + * unlike(got, """expected""".r, testName) + * + * skip(howMany, why) { + * ok(foo(), testName) + * is(foo(42), 23, testName) + * } + * + * todo(why) { + * ok(foo(), testName) + * is(foo(42), 23, testName) + * } + * + * pass(testName) + * fail(testName) + * + * BAIL_OUT(why) + * } + * </pre> + * + * The difference between the simple `ok` method and the more specific + * methods like `is` and `like` is in how failures are reported. If you write + * this: + * + * <pre> + * ok(1 == 2) + * </pre> + * + * the output will look like this: + * + * <pre> + * not ok 1 + * # Failed test at MyTest.scala line 4. + * </pre> + * + * On the other hand, a more specific assertion such as: + * + * <pre> + * is(1, 2) + * </pre> + * + * will produce more useful output: + * + * <pre> + * not ok 1 + * # Failed test at MyTest.scala line 4. + * # got: 1 + * # expected: 2 + * </pre> + * + * In addition to assertions, there are also several methods which take a + * block of code to run, to modify the assertions contained in that block. + * + * The `todo` method runs tests which are expected to fail. If they do fail, + * the failure is reported to the test harness as a normal succeeding test, + * and nothing happens. If they succeed, they are still reported as a + * succeeding test, but a message is displayed to the user indicating that + * the todo indication can be removed. + * + * The `skip` method takes a block which should not be run at all. This is + * similar to `todo`, except that it is useful for tests which could cause + * problems if they were to actually run. Since the tests are never run, it's + * not possible to count how many tests there should be, so this must be + * specified as a parameter. + * + * The `subtest` method runs a block of assertions as though they were an + * entirely separate test, and then reports the result of that test as a + * single assertion in the test that called `subtest`. + * + * ==Test plans== + * + * Normally, you can run any number of assertions within your class body, and + * the framework will assume that if no exceptions were thrown, all of the + * assertions that were meant to be run were actually run. Sometimes, + * however, that may not be a safe assumption, especially with heavily + * callback-driven code. In this case, you can specify exactly how many tests + * you intend to run, and the number of tests actually run will be checked + * against this at the end. To declare this, give a number to the `TestMore` + * constructor: + * + * <pre> + * class MyTest extends TestMore(5) { + * ??? + * } + * </pre> + * + * In addition, if the entire test should be skipped, you can give a plan of + * `SkipAll()`: + * + * <pre> + * class MyTest extends TestMore(SkipAll()) { + * ??? + * } + * </pre> + * + * ==Extensions== + * + * These assertion methods are written with the intention of being + * composable. You can write your own test methods which call `is` or `ok` on + * more specific bits of data. The one issue here is that, as shown above, + * test failure messages refer to the file and line where the `is` or `ok` + * call was made. If you want this to instead point at the line where your + * assertion helper method was called, you can use the `hideTestMethod` + * method like this: + * + * <pre> + * trait MyTestHelpers { this: TestMore => + * def notok (cond: Boolean) = hideTestMethod { + * ok(!cond) + * } + * } + * </pre> + * + * This way, the test failure will be reported from the line where `notok` + * was called, not from the call to `ok` in the `notok` method. + */ +class TestMore (plan: Plan = NoPlan) extends Test with DelayedInit { + def delayedInit (body: => Unit) { + testBody = { terminalInUse => + todo = None + builder = new TestBuilder(plan, terminalInUse) + plan match { + case SkipAll(_) => () + case _ => body + } + } + } + + protected def runTests (terminalInUse: Boolean): Int = { + if (testBody == null) { + delayedInit { } + } + + testBody(terminalInUse) + builder.doneTesting + builder.exitCode + } + + /** Assert that a condition is true. + * + * @example `ok(response.isSuccess)` + */ + def ok (cond: Boolean): Boolean = + test(cond) + + /** Assert that a condition is true, and describe the assertion. + * + * @example `ok(response.isSuccess, "the response succeeded")` + */ + def ok (cond: Boolean, desc: String): Boolean = + testWithDesc(cond, desc) + + /** Assert that two objects are equal (using `==`). + * + * @example `is(response.status, 200)` + */ + def is[T] (got: T, expected: T): Boolean = + test(got == expected, isMessage(got, expected)) + + /** Assert that two objects are equal (using `==`), and describe the + * assertion. + * + * @example `is(response.status, 200, "we got a 200 OK response")` + */ + def is[T] (got: T, expected: T, desc: String): Boolean = + testWithDesc(got == expected, desc, isMessage(got, expected)) + + /** Assert that two objects are not equal (using `!=`). + * + * @example `isnt(response.body, "")` + */ + def isnt[T] (got: T, expected: T): Boolean = + test(got != expected, isntMessage(got)) + + /** Assert that two objects are not equal (using `!=`), and describe the + * assertion. + * + * @example `isnt(response.body, "", "we got a response body")` + */ + def isnt[T] (got: T, expected: T, desc: String): Boolean = + testWithDesc(got != expected, desc, isntMessage(got)) + + /** Assert that a string matches a regular expression. + * + * @example `like(response.header("Content-Type"), """text/x?html""".r)` + */ + def like (got: String, rx: Regex): Boolean = + test(rx.findFirstIn(got).nonEmpty, likeMessage(got, rx)) + + /** Assert that a string matches a regular expression, and describe the + * assertion. + * + * @example `like(response.header("Content-Type"), """text/x?html""".r, "we got an html content type")` + */ + def like (got: String, rx: Regex, desc: String): Boolean = + testWithDesc(rx.findFirstIn(got).nonEmpty, desc, likeMessage(got, rx)) + + /** Assert that a string doesn't match a regular expression. + * + * @example `unlike(response.header("Authorization"), """^Digest.*""".r)` + */ + def unlike (got: String, rx: Regex): Boolean = + test(rx.findFirstIn(got).isEmpty, unlikeMessage(got, rx)) + + /** Assert that a string doesn't match a regular expression. + * + * @example `unlike(response.header("Authorization"), """^Digest.*""".r, "we don't support digest authentication")` + */ + def unlike (got: String, rx: Regex, desc: String): Boolean = + testWithDesc(rx.findFirstIn(got).isEmpty, desc, unlikeMessage(got, rx)) + + /** An assertion that always succeeds. + * + * @example `pass()` + */ + def pass: Boolean = + ok(true) + + /** An assertion that always succeeds, with a reason. + * + * @example `pass("this line of code should be executed")` + */ + def pass (desc: String): Boolean = + ok(true, desc) + + /** An assertion that always fails. + * + * @example `fail()` + */ + def fail: Boolean = + ok(false) + + /** An assertion that always fails, with a reason. + * + * @example `fail("we should never get here")` + */ + def fail (desc: String): Boolean = + ok(false, desc) + + /** Output a comment to `Console.err`. This is intended to be visible to + * users even when running the test under a summarizing harness. + * + * @example `diag("Testing with Scala " + util.Properties.versionString)` + */ + def diag (message: String) { + builder.diag(message) + } + + /** Output a comment to `Console.out`. This is intended to only be visible + * when viewing the raw TAP stream. + * + * @example `note("Starting the response tests")` + */ + def note (message: String) { + builder.note(message) + } + + /** Halt execution of the entire test suite. + * + * @example `BAIL_OUT("can't connect to the database!")` + */ + def BAIL_OUT (desc: String) { + builder.bailOut(desc) + } + + /** Mark a block of tests as expected to fail. If the tests which run in the + * todo block fail, they will not be treated as test failures, and if they + * succeed, the user will be notified. + * + * @example `todo("waiting on fixes elsewhere") { ??? }` + */ + def todo (reason: String)(body: => Unit) { + val oldTodo = todo + try { + todo = Some(reason) + body + } + finally { + todo = oldTodo + } + } + + /** Mark a block of tests that should not be run at all. They are treated as + * always passing. + * + * @example `skip(3, "too dangerous to run for now") { ??? }` + */ + def skip (count: Int, reason: String)(body: => Unit) { + for (i <- 1 to count) { + builder.skip(reason) + } + } + + /** Declare a logical group of assertions, to be run as a single test. This + * is effectively an entirely separate test, which is run, and the result + * of that test is reported as a single assertion in the test that contains + * it. The subtest can specify its own plan in the same way that the + * overall test is allowed to. The name will be used as the description for + * the single assertion that the overall test sees. + * + * @example `subtest("response tests") { ??? }` + */ + def subtest ( + name: String, + plan: Plan = NoPlan + )(body: => Unit): Boolean = { + val oldBuilder = builder + val success = try { + builder = oldBuilder.cloneForSubtest(plan) + body + builder.doneTesting + } + finally { + builder = oldBuilder + } + ok(success, name) + } + + /** A helper method which should be used to wrap test utility methods. + * Normally, when tests fail, a message is printed giving the file and line + * number of the call to the test method. If you write your own test + * methods, they will typically use the existing methods to generate + * assertions, and so the file and line numbers will likely be much less + * useful. Wrapping the body of your method in this method will ensure that + * the file and line number that is reported is the line where your helper + * method is called instead. + * + * @example `def testFixtures = hideTestMethod { ??? }` + */ + def hideTestMethod[T] (body: => T): T = { + // this just adds a method call with a known name to the stack trace, so + // that we can detect it later + body + } + + private def isMessage[T] (got: T, expected: T): String = + " got: '" + got + "'\n" + + " expected: '" + expected + "'\n" + + private def isntMessage[T] (got: T): String = + " got: '" + got + "'\n" + + " expected: anything else\n" + + private def likeMessage (got: String, rx: Regex): String = + " '" + got + "'\n" + + " doesn't match '" + rx + "'\n" + + private def unlikeMessage (got: String, rx: Regex): String = + " '" + got + "'\n" + + " matches '" + rx + "'\n" + + private def testWithDesc ( + cond: Boolean, + desc: String + ): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond, "- " + desc) + case None => builder.ok(cond, "- " + desc) + } + if (!cond) { + failed(Some(desc), None) + } + cond + } + + private def testWithDesc ( + cond: Boolean, + desc: String, + reason: => String + ): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond, "- " + desc) + case None => builder.ok(cond, "- " + desc) + } + if (!cond) { + failed(Some(desc), Some(reason)) + } + cond + } + + private def test (cond: Boolean): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond) + case None => builder.ok(cond) + } + if (!cond) { + failed(None, None) + } + cond + } + + private def test (cond: Boolean, reason: => String): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond) + case None => builder.ok(cond) + } + if (!cond) { + failed(None, Some(reason)) + } + cond + } + + private def failed (desc: Option[String], reason: Option[String]) { + val stack = Thread.currentThread.getStackTrace.drop(1).filter { frame => + !ignoreFrame(frame) + } + val idx = stack.lastIndexWhere { frame => + frame.getClassName == "com.iinteractive.test.TestMore" && + frame.getMethodName == "hideTestMethod" + } + val caller = idx match { + case -1 => stack.headOption + // one level to jump out of hideTestMethod and one level to jump out of + // the method that called hideTestMethod + case i => stack.drop(i + 2).headOption + + } + val (file, line) = caller match { + case Some(frame) => (frame.getFileName, frame.getLineNumber) + case None => ("<unknown file>", "<unknown line>") + } + val message = " " + (todo match { + case Some(_) => "Failed (TODO) test" + case None => "Failed test" + }) + (desc match { + case Some(m) => " '" + m + "'\n " + case None => " " + }) + val trace = "at " + file + " line " + line + "." + val explanation = message + trace + reason.map("\n" + _).getOrElse("") + if (todo.isDefined) { + builder.note(explanation) + } + else { + builder.diag(explanation) + } + } + + protected def ignoreFrame (frame: StackTraceElement): Boolean = { + val className = frame.getClassName + val methodName = frame.getMethodName + + // ignore everything in this class, except the hideTestMethod call which we + // use as a stack trace marker + (className == "com.iinteractive.test.TestMore" && + methodName != "hideTestMethod") || + // when you call a method in a class when the method is defined in a + // trait, it calls a stub which calls the real definition in the trait. + // the trait is represented under the hood as a class with the same name + // as the trait, except with $class appended. this is a gross reliance on + // implementation details that could change at any moment, but i don't + // really see any better options. + """\$class$""".r.findFirstIn(className).nonEmpty + } + + private var todo: Option[String] = _ + private var builder: TestBuilder = _ + private var testBody: Boolean => Unit = _ +} diff --git a/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala new file mode 100644 index 0000000..56f32fd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala @@ -0,0 +1,16 @@ +package com.iinteractive.test.harness + +/** Classes that implement `MultiTestReporter` are capable of running a group + * of test classes, given their names. This typically involves some sort of + * summarization. + * + * @see [[com.iinteractive.test.harness.Reporter Reporter]]. + */ +trait MultiTestReporter { + /** Runs the test classes identifed by the list of fully qualified class + * names `testNames`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testNames: Seq[String]): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/Reporter.scala b/src/main/scala/com/iinteractive/test/harness/Reporter.scala new file mode 100644 index 0000000..a47444b --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/Reporter.scala @@ -0,0 +1,15 @@ +package com.iinteractive.test.harness + +/** Classes that implement `Reporter` are capable of running a test class, + * given its name. + * + * @see [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]]. + */ +trait Reporter { + /** Runs the test class identifed by the fully qualified class name + * `testName`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testName: String): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala new file mode 100644 index 0000000..cd555dd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala @@ -0,0 +1,42 @@ +package com.iinteractive.test.harness + +import java.io.{PipedInputStream,PipedOutputStream} +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +import com.iinteractive.test.tap.{Parser,TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.Test + +/** This is a trait for classes that run tests and summarize the results. It + * provides a single `runOneTest` method, which runs a test class and + * produces a stream of [[com.iinteractive.test.tap.TAPEvent TAP events]] + * which can be used to produce whatever summarized output you need. + */ +trait SummarizedTests { + /** Runs a single [[com.iinteractive.test.Test test]] instance, calling `cb` + * with each [[com.iinteractive.test.tap.TAPEvent TAP event]] as it is + * produced. + * + * @return The overall result of the test instance. + */ + protected def runOneTest (test: Test, cb: TAPEvent => Unit): TAPResult = { + val out = new PipedOutputStream + val in = new PipedInputStream(out) + + val testFuture = Future { + Console.withOut(out) { + test.runInHarness + } + out.close + } + + val parser = new Parser(cb) + val result = parser.parse(in) + in.close + Await.ready(testFuture, Duration.Inf) + + result + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala new file mode 100644 index 0000000..a5fe1e0 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala @@ -0,0 +1,192 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap.{TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.tap.{StartEvent,ResultEvent,PlanEvent,EndEvent} +import com.iinteractive.test.Test + +/** Runs a series of tests. The TAP output from these tests is parsed, and + * output is produced which is similar in style to Perl's + * [[https://metacpan.org/module/Test::Harness Test::Harness]]. + */ +class SummaryReporter extends MultiTestReporter with SummarizedTests { + def run (testNames: Seq[String]): Int = { + val results = runTests(testNames) + val success = results.values.forall(_.success) + printTestSummary(success, results) + if (success) 0 else 1 + } + + protected def runTests (testNames: Seq[String]): Map[String, TAPResult] = { + val maxLength = testNames.map(_.length).max + + testNames.map { name => + val callbackGenerator: () => TAPEvent => Unit = () => { + var width = 0 + var tests = 0 + var plan: Option[Int] = None + + def status = { + tests + "/" + plan.getOrElse("?") + } + + def printStatus (st: String) { + print("\r" + (" " * width) + "\r") + val line = + name + " " + ("." * (maxLength - name.length)) + ".. " + st + width = line.length + print(line) + Console.out.flush + } + + (e: TAPEvent) => e match { + case StartEvent => { + printStatus("") + } + case PlanEvent(p) => { + plan = Some(p.plan) + printStatus(status) + } + case ResultEvent(r) => { + tests += 1 + printStatus(status) + } + case EndEvent(result) => { + if (result.success) { + printStatus("") + println("ok") + } + else { + val results = result.results.length + val failed = result.results.count { t => + !t.passed && !t.directive.isDefined + } + + printStatus("") + println("Dubious, test returned " + result.exitCode) + println("Failed " + failed + "/" + results + " subtests") + } + } + case _ => () + } + } + + name -> runOneTest(newInstance[Test](name), callbackGenerator()) + }.toMap + } + + protected def printTestSummary ( + success: Boolean, + results: Map[String, TAPResult] + ) { + printSuccess(success) + printLongSummary(results) + printShortSummary(results) + printPassFail(success, results) + } + + private def printSuccess (success: Boolean) { + if (success) { + println("All tests successful.") + } + } + + private def printShortSummary (results: Map[String, TAPResult]) { + val files = results.size + val tests = results.values.map(_.results.length).sum + println("Files=" + files + ", Tests=" + tests) + } + + private def printLongSummary (results: Map[String, TAPResult]) { + val todoSucceeded = results.mapValues { r => + r.results.filter { t => + t.directive match { + case Some(TodoDirective(_)) => t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testsFailed = results.mapValues { r => + r.results.filter { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testNames = (todoSucceeded ++ testsFailed).keys + + if (testNames.nonEmpty) { + println("") + println("Test Summary Report") + println("-------------------") + + val maxLength = testNames.map(_.length).max + + for (name <- testNames) { + val result = results(name) + + println( + name + (" " * (maxLength - name.length)) + " " + + "(Tests: " + result.results.length + " " + + "Failed: " + testsFailed.getOrElse(name, Nil).length + ")" + ) + + if (testsFailed.isDefinedAt(name)) { + val fails = testsFailed(name) + println( + " Failed test" + (if (fails.length > 1) "s" else "") + ": " + + fails.map(_.number).mkString(", ") + ) + } + + if (todoSucceeded.isDefinedAt(name)) { + val todos = todoSucceeded(name) + println( + " TODO passed: " + + todos.map(_.number).mkString(", ") + ) + } + + val exitCode = results(name).exitCode + if (exitCode != 0) { + println(" Non-zero exit status: " + exitCode) + } + } + } + } + + private def printPassFail ( + success: Boolean, + results: Map[String, TAPResult] + ) { + if (success) { + println("Result: PASS") + } + else { + println("Result: FAIL") + + val testResults = results.values + + val testsFailed = testResults.map { r => + r.results.count { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_ > 0) + val failedFiles = testsFailed.size + val failedTests = testsFailed.sum + + val allFiles = testResults.size + val allTests = testResults.map(_.results.length).sum + + println( + "Failed " + failedFiles + "/" + allFiles + " test programs. " + + failedTests + "/" + allTests + " subtests failed." + ) + } + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala new file mode 100644 index 0000000..8a4dc9f --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala @@ -0,0 +1,12 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap +import com.iinteractive.test.Test + +/** Runs a single test. The TAP stream from that test is written directly to + * stdout/stderr. + */ +class TAPReporter extends Reporter { + def run (testName: String): Int = + newInstance[Test](testName).run +} diff --git a/src/main/scala/com/iinteractive/test/harness/TestHarness.scala b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala new file mode 100644 index 0000000..7b8f4ae --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala @@ -0,0 +1,110 @@ +package com.iinteractive.test.harness + +/** This is the entry point to running tests written with this library from + * the command line. Note that this library also implements the + * [[https://github.com/harrah/test-interface common testing interface]] for + * test libraries, so tests should also just work with `sbt test`. + * + * If this application is run and given just a single test class name, it + * will run that test and write its TAP stream to the console. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest + * ok 1 + * ok 2 + * 1..2 + * }}} + * + * If this application is run and given multiple test class names, it will + * run each of those tests, and present a summary report, similar to the one + * produces by + * [[https://metacpan.org/module/Test::Harness Perl's Test::Harness]]. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest1 MyTest2 + * MyTest1 .. ok + * MyTest2 .. ok + * All tests successful. + * Files=2, Tests=4 + * Result: PASS + * }}} + * + * This application also accepts a few command line options to customize its + * behavior: + * + * - `-r`: Alternative [[com.iinteractive.test.harness.Reporter Reporter]] + * class to use for running a single test. + * - `-R`: Alternative + * [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]] + * class to use for running a group of tests. Also enables using the + * MultiTestReporter for a single test. + * - `--help`: Prints usage information. + */ +object TestHarness { + import com.iinteractive.test.Test + + /** Entry point for the harness application. */ + def main (args: Array[String]) { + val opts = parseOpts(args.toList) + val single = opts("prefer-single").asInstanceOf[Boolean] + + val exitCode = if (single) { + val reporterName = opts("single-reporter").asInstanceOf[String] + val testName = opts("test-classes").asInstanceOf[List[String]].apply(0) + val reporter = newInstance[Reporter](reporterName) + reporter.run(testName) + } + else { + val reporterName = opts("multi-reporter").asInstanceOf[String] + val testNames = opts("test-classes").asInstanceOf[List[String]] + val reporter = newInstance[MultiTestReporter](reporterName) + reporter.run(testNames) + } + + sys.exit(exitCode) + } + + protected def parseOpts (args: List[String]): Map[String, Any] = args match { + case Nil => Map( + "single-reporter" -> "com.iinteractive.test.harness.TAPReporter", + "multi-reporter" -> "com.iinteractive.test.harness.SummaryReporter", + "prefer-single" -> true, + "test-classes" -> Nil + ) + + case "-r" :: singleReporter :: rest => + parseOpts(rest) + ("single-reporter" -> singleReporter) + + case "-R" :: multiReporter :: rest => + parseOpts(rest) ++ Map( + "multi-reporter" -> multiReporter, + "prefer-single" -> false + ) + + case "--help" :: rest => + usage(0) + + case `unknownOption` :: rest => + usage(1) + + case testClass :: rest => { + val opts = parseOpts(rest) + val tests = opts("test-classes").asInstanceOf[List[String]] + opts ++ Map( + "test-classes" -> (testClass :: tests), + "prefer-single" -> tests.isEmpty + ) + } + } + + protected def usage (exitCode: Int) = { + val out = if (exitCode == 0) Console.out else Console.err + out.println("harness [-r <single-reporter-class>]\n" + + " [-R <multi-reporter-class>]\n" + + " [--help]\n" + + " <test-class> [<test-class>...]\n") + sys.exit(exitCode) + } + + private val unknownOption = """^-.*""".r +} diff --git a/src/main/scala/com/iinteractive/test/harness/package.scala b/src/main/scala/com/iinteractive/test/harness/package.scala new file mode 100644 index 0000000..1b74f9d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/package.scala @@ -0,0 +1,33 @@ +package com.iinteractive.test + +/** Classes to handle running test instances and providing output. */ +package object harness { + import scala.reflect.{ClassTag,classTag} + + /** Loads `className`, returning the + * [[http://docs.oracle.com/javase/7/docs/api/java/lang/Class.html Class]] + * instance. + */ + def loadClass[T: ClassTag] (className: String): Class[_] = + classTag[T].runtimeClass.getClassLoader.loadClass(className) + + /** Loads `className` and creates a new instance of it, using the + * no-argument constructor. + */ + def newInstance[T: ClassTag] (className: String): T = + loadClass[T](className).newInstance.asInstanceOf[T] + + /** Loads `className` and creates a new instance of it, using a + * one-argument constructor. Passes `arg` as the argument to the + * constructor. + */ + def newInstance[T: ClassTag, U <: AnyRef: ClassTag] ( + className: String, + arg: U + ): T = { + val classObj = loadClass[T](className) + val argClassObj = classTag[U].runtimeClass + val constructor = classObj.getConstructor(argClassObj) + constructor.newInstance(arg).asInstanceOf[T] + } +} diff --git a/src/main/scala/com/iinteractive/test/package.scala b/src/main/scala/com/iinteractive/test/package.scala new file mode 100644 index 0000000..df077f2 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/package.scala @@ -0,0 +1,74 @@ +package com.iinteractive + +/** This library implements several components of Perl's testing ecosystem in + * Scala. The most useful place to get started to use this library is likely + * [[com.iinteractive.test.TestMore TestMore]]. + */ +package object test { + import language.implicitConversions + + /** Converts an + * [[http://www.scala-lang.org/api/current/index.html#scala.Int Int]] to a + * [[NumericPlan]]. + */ + implicit def intToPlan (p: Int): Plan = + NumericPlan(p) + + /** A test plan. This represents the TAP statement telling how many tests + * will be run. + */ + sealed trait Plan { + /** How many tests will be run. */ + val plan: Int + /** Whether this test was skipped. It should contain `Some(message)` if + * the test is skipped, and `None` otherwise. + */ + val skipAll: Option[String] + } + + /** An explicit plan number. Corresponds to `1..5` in TAP. */ + case class NumericPlan (plan: Int) extends Plan { + /** @inheritdoc + * + * Always `None` for this class. + */ + val skipAll = None + } + + /** A test which did not declare a plan yet. */ + case object NoPlan extends Plan { + /** @inheritdoc + * + * Always 0 for this class. + */ + val plan = 0 + /** @inheritdoc + * + * Always `None` for this class. + */ + val skipAll = None + } + + /** A test which has declared that the entire test has been skipped. + * Corresponds to `1..0 # SKIP [message]` in TAP. + */ + case class SkipAll (message: String) extends Plan { + /** @inheritdoc + * + * Always 0 for this class. + */ + val plan = 0 + /** @inheritdoc + * + * Never `None` for this class. + */ + val skipAll = Some(message) + } + + /** Exception thrown when a test bails out. Corresponds to + * `Bail out! [message]` in TAP. + */ + case class BailOutException ( + message: String + ) extends RuntimeException(message) +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala b/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala new file mode 100644 index 0000000..bab13c5 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala @@ -0,0 +1,11 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Fingerprint.java org.scalatools.testing.Fingerprint]]. + */ +object Fingerprint extends testing.SubclassFingerprint { + def isModule: Boolean = false + def superClassName: String = "com.iinteractive.test.Test" +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Framework.scala b/src/main/scala/com/iinteractive/test/sbt/Framework.scala new file mode 100644 index 0000000..bb3d0bb --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Framework.scala @@ -0,0 +1,18 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Framework.java org.scalatools.testing.Framework]]. + */ +class Framework extends testing.Framework { + val name: String = "Perl8 Test" + val tests: Array[testing.Fingerprint] = Array(Fingerprint) + + def testRunner ( + testClassLoader: ClassLoader, + loggers: Array[testing.Logger] + ): testing.Runner = { + new Runner(testClassLoader, loggers) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Runner.scala b/src/main/scala/com/iinteractive/test/sbt/Runner.scala new file mode 100644 index 0000000..0eee4cf --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Runner.scala @@ -0,0 +1,25 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +import com.iinteractive.test.harness.SummaryReporter +import com.iinteractive.test.Test + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Runner2.java org.scalatools.testing.Runner2]] + * using [[com.iinteractive.test.sbt.SBTReporter SBTReporter]]. + */ +class Runner ( + loader: ClassLoader, + loggers: Array[testing.Logger] +) extends testing.Runner2 { + def run ( + testClassName: String, + fingerprint: testing.Fingerprint, + eventHandler: testing.EventHandler, + args: Array[String] + ) { + val reporter = new SBTReporter(loader, loggers, eventHandler) + reporter.run(testClassName) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala b/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala new file mode 100644 index 0000000..34df60d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala @@ -0,0 +1,99 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +import com.iinteractive.test.harness.{Reporter,SummarizedTests} +import com.iinteractive.test.tap.{TAPEvent,ResultEvent,EndEvent} +import com.iinteractive.test.Test + +/** Runs a single test under the SBT test harness. */ +class SBTReporter ( + loader: ClassLoader, + loggers: Array[testing.Logger], + eventHandler: testing.EventHandler +) extends Reporter with SummarizedTests { + def run (testName: String): Int = { + val cb = (e: TAPEvent) => e match { + case ResultEvent(r) => { + val event = new testing.Event { + val testName: String = r.description + val description: String = r.description + val result: testing.Result = + if (r.passed) { + testing.Result.Success + } + else if (r.directive.isDefined) { + testing.Result.Skipped + } + else { + testing.Result.Failure + } + val error: Throwable = null + } + eventHandler.handle(event) + } + case EndEvent(result) => { + val testsPassed = result.success + val correctCode = result.exitCode == 0 + val event = new testing.Event { + val testName: String = "exit code is 0" + val description: String = "exit code is 0" + val result: testing.Result = + if (correctCode) { + testing.Result.Success + } + else { + testing.Result.Failure + } + val error: Throwable = null + } + eventHandler.handle(event) + + if (testsPassed && correctCode) { + logInfo("PASS " + testName) + } + else { + val results = result.results.length + val failed = result.results.count { t => + !t.passed && !t.directive.isDefined + } + + val errors = Seq( + (if (testsPassed) + None + else + Some("failed " + failed + "/" + results)), + (if (correctCode) + None + else + Some("non-zero exit code: " + result.exitCode)) + ).flatten.mkString("(", ", ", ")") + + logError("FAIL " + testName + " " + errors) + } + } + case _ => () + } + + runOneTest( + loader.loadClass(testName).newInstance.asInstanceOf[Test], + cb + ).exitCode + } + + private def logDebug (msg: String) { + loggers.foreach(_.debug(msg)) + } + + private def logInfo (msg: String) { + loggers.foreach(_.info(msg)) + } + + private def logWarn (msg: String) { + loggers.foreach(_.warn(msg)) + } + + private def logError (msg: String) { + loggers.foreach(_.error(msg)) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/package.scala b/src/main/scala/com/iinteractive/test/sbt/package.scala new file mode 100644 index 0000000..eeb1f68 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/package.scala @@ -0,0 +1,4 @@ +package com.iinteractive.test + +/** Classes for interoperating with `sbt test`. */ +package object sbt diff --git a/src/main/scala/com/iinteractive/test/tap/Consumer.scala b/src/main/scala/com/iinteractive/test/tap/Consumer.scala new file mode 100644 index 0000000..09a2f0a --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Consumer.scala @@ -0,0 +1,131 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** Contains a method to parse an individual line of TAP. */ +object Consumer { + /** Parses a line of TAP. + * + * @return A [[com.iinteractive.test.tap.Consumer.Line Line]] object + * corresponding to the parsed line. + */ + def parseLine (line: String): Line = { + commentRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, text) => new CommentLine(text, indent) + } + }.getOrElse { + planRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, p, null) => + new PlanLine(NumericPlan(p.toInt), indent) + case Seq(indent, _, skip) => + new PlanLine(SkipAll(skip), indent) + } + }.getOrElse { + resultRx.findFirstMatchIn(line).map { m => + val indent = m.group(1) + val passed = m.group(2) == null + val number = m.group(3).toInt + val description = m.group(4) match { + case null => "" + case s => s.trim + } + val directive = (m.group(5), m.group(6)) match { + case (null, null) => None + case (d, r) => { + val reason = if (r == null) "" else r + """(?i:skip)""".r.findFirstIn(d) match { + case Some(_) => Some(new SkipDirective(Some(reason))) + case None => Some(new TodoDirective(Some(reason))) + } + } + } + val result = new TestResult( + passed, + number, + description, + directive, + None + ) + new ResultLine(result, indent) + }.getOrElse { + throw ParseException("Couldn't parse line: " + line) + } + } + } + } + + /** The representation of a parsed line of TAP. */ + sealed trait Line { + /** The meaningful portion of the TAP line. */ + def contents: String + /** The indentation of the TAP line (used in subtests). */ + def indent: String + + /** The line itself that was parsed. */ + override def toString: String = + indent + contents + } + + /** A parsed TAP line containing a comment. + * + * @param text The text of the comment (not including the `#`). + */ + case class CommentLine private[Consumer] ( + text: String, + indent: String + ) extends Line { + def contents = "# " + text + } + + /** A parsed TAP line containing a test plan. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] that this line + * represents. + */ + case class PlanLine private[Consumer] ( + plan: Plan, + indent: String + ) extends Line { + def contents = { + val count = plan.plan + val comment = plan match { + case SkipAll(m) => " # SKIP " + m + case _ => "" + } + indent + "1.." + count + comment + } + } + + /** A parsed TAP line containing a test result. + * + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * that this line represents. + */ + case class ResultLine private[Consumer] ( + result: TestResult, + indent: String + ) extends Line { + def contents = { + val success = (if (result.passed) "ok" else "not ok") + " " + val number = result.number + " " + val description = result.description match { + case "" => "" + case s => s + " " + } + val directive = result.directive.map { d => + d match { + case TodoDirective(m) => "# TODO " + m + case SkipDirective(m) => "# skip " + m + } + }.getOrElse("") + indent + success + number + description + directive + } + } + + private val commentRx = """^(\s*)#\s*(.*)""".r + private val planRx = """^(\s*)1..(\d+)\s*(?:# SKIP (.*))?""".r + private val resultRx = + """^(\s*)(not )?ok (\d+)\s*([^#]+)?(?:#\s*(?i:(skip|todo))\s+(.*))?""".r +} diff --git a/src/main/scala/com/iinteractive/test/tap/Parser.scala b/src/main/scala/com/iinteractive/test/tap/Parser.scala new file mode 100644 index 0000000..7bc44d3 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Parser.scala @@ -0,0 +1,232 @@ +package com.iinteractive.test.tap + +import java.io.{ByteArrayInputStream,InputStream,OutputStream} +import scala.annotation.tailrec +import scala.io.Source +import scala.util.parsing.combinator._ +import scala.util.parsing.input.{Position,Reader} + +import com.iinteractive.test.Plan +import com.iinteractive.test.tap.Consumer._ + +/** This class parses a TAP stream. It can either parse it all at once (from a + * string), or it can be used as a streaming parser, where TAP events are + * emitted through a given callback. + */ +class Parser private ( + cb: TAPEvent => Unit, + indent: String +) { + /** Creates a parser instance. + * @param cb The event handler callback. It will be called after each + * meaningful line of TAP, with a + * [[com.iinteractive.test.tap.TAPEvent TAPEvent]] instance + * representing the event that was just parsed. + */ + def this (cb: TAPEvent => Unit = e => ()) = + this(cb, "") + + private def this (indent: String) = + this(e => (), indent) + + /** Parses TAP from an input stream. This variant will actually parse lines + * as they are available to read from the input stream, so this can be used + * as a streaming parser. + */ + def parse (input: InputStream): TAPResult = { + import parser._ + + cb(StartEvent) + tap(new LineReader(input)) match { + case Success(result, _) => { + cb(EndEvent(result)) + result + } + case failure: NoSuccess => throw new ParseException(failure.msg) + } + } + + /** Parses TAP contained in a string. This isn't useful for incremental + * parsing, because the entire input string must be created before + * parsing can begin. + */ + def parse (input: String): TAPResult = + parse(new ByteArrayInputStream(input.getBytes)) + + /** Parses TAP from an output stream. + * + * @todo Doesn't currently work as a streaming parser, since it just + * collects the entire output as a string and feeds it to the parser + * for strings. This could likely be improved. + */ + def parse (input: OutputStream): TAPResult = + parse(input.toString) + + private val parser = new TAPParser(cb, indent) + + private class TAPParser ( + cb: TAPEvent => Unit, + indent: String + ) extends Parsers { + type Elem = Line + + def tap: Parser[TAPResult] = + planFirst | planLast + + private def planFirst: Parser[TAPResult] = + plan ~ rep(result) ^^ { case plan ~ results => + new TAPResult(plan, results) + } + + private def planLast: Parser[TAPResult] = + rep(result) ~ plan ^^ { case results ~ plan => + new TAPResult(plan, results) + } + + private def plan: Parser[Plan] = + planLine ^^ { p => + cb(PlanEvent(p.plan)) + p.plan + } + + private def result: Parser[TestResult] = + simpleResult | subtestResult + + private def simpleResult: Parser[TestResult] = + resultLine ^^ { r => + cb(ResultEvent(r.result)) + r.result + } + + private def subtestResult: Parser[TestResult] = + subtest ~ simpleResult ^^ { case subtest ~ simpleResult => + new TestResult( + simpleResult.passed, + simpleResult.number, + simpleResult.description, + simpleResult.directive, + Some(subtest) + ) + } + + private def subtest: Parser[TAPResult] = + LineParser("subtest") { in => + // can't just return the result directly, because it's of a different + // type (the path dependent type associated with the new Parser + // instance we create here, rather than the path dependent type + // associated with this) + val subParser = new TAPParser( + e => (), + in.first.indent + ) + subParser.tap(in) match { + case subParser.Success(p, rest) => Success(p, rest) + case subParser.Failure(m, rest) => Failure(m, rest) + case subParser.Error(m, rest) => Error(m, rest) + } + } + + private def planLine: Parser[PlanLine] = LineParser("plan") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: PlanLine => + Success(p, in.rest) + case _ => + Failure("Plan line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Plan line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def resultLine: Parser[ResultLine] = LineParser("result") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: ResultLine => + Success(p, in.rest) + case _ => + Failure("Result line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Result line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def LineParser[T] (lineType: String)( + body: Input => ParseResult[T] + ): Parser[T] = Parser { in => + if (in.atEnd) { + Failure(lineType + " line expected, but end of input found", in) + } + else { + body(in) + } + } + } + + private class LineReader ( + in: Iterator[Char], + lineNum: Int + ) extends Reader[Line] { + def this (in: InputStream) = + this(Source.fromInputStream(in), 1) + + def atEnd: Boolean = + nextLine.isEmpty + + def first: Line = + nextLine.getOrElse(throw new RuntimeException("read from empty input")) + + lazy val pos = + new LinePosition(lineNum, nextLine.map(_.toString).getOrElse("")) + + lazy val rest: Reader[Line] = + new LineReader(remainingStream, lineNum + 1) + + private def nextLine: Option[Line] = + state._1 + + private def remainingStream: Iterator[Char] = + state._2 + + private lazy val state: (Option[Line], Iterator[Char]) = + readNextLine(in) + + @tailrec + private def readNextLine ( + stream: Iterator[Char] + ): (Option[Line], Iterator[Char]) = { + if (stream.hasNext) { + val (line, rest) = stream.span(_ != '\n') match { + case (l, r) => (parseLine(l.mkString), r.drop(1)) + } + line match { + case _: CommentLine => readNextLine(rest) + case other => (Some(other), rest) + } + } + else { + (None, in) + } + } + } + + private case class LinePosition ( + line: Int, + lineContents: String + ) extends Position { + def column: Int = 1 + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/Producer.scala b/src/main/scala/com/iinteractive/test/tap/Producer.scala new file mode 100644 index 0000000..3ed8d7e --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Producer.scala @@ -0,0 +1,63 @@ +package com.iinteractive.test.tap + +/** Contains functions for producing individual lines of TAP. */ +object Producer { + import com.iinteractive.test.Plan + + /** Returns a test result. + * + * @example `ok 4` + */ + def result (cond: Boolean, num: Int): String = + (if (cond) "ok " else "not ok ") + num + + /** Returns a test result that contains a description. + * + * @example `ok 28 - our test succeeded` + */ + def result (cond: Boolean, num: Int, desc: String): String = + result(cond, num) + " " + desc + + /** Returns a todo test result. + * + * @example `not ok 1 # TODO this doesn't work yet` + */ + def todoResult (cond: Boolean, num: Int, todo: String): String = + result(cond, num) + " # TODO " + todo + + /** Returns a todo test result that contains a description. + * + * @example `not ok 18 - test the feature # TODO can't figure this out` + */ + def todoResult (cond: Boolean, num: Int, desc: String, todo: String): String = + result(cond, num, desc) + " # TODO " + todo + + /** Returns a skipped test result with a reason. + * + * @example `ok 4 # skip this test won't run here` + */ + def skip (num: Int, reason: String): String = + "ok " + num + " # skip " + reason + + /** Returns a comment. + * + * @example `# this is a comment` + */ + def comment (message: String): String = + message.split("\n").map(m => "# " + m).mkString("\n") + + /** Returns a test plan. + * + * @example `1..5` ([[com.iinteractive.test.NumericPlan NumericPlan]]) + * @example `1..0 # SKIP don't run this test` ([[com.iinteractive.test.SkipAll SkipAll]]) + */ + def plan (plan: Plan): String = + plan.skipAll.map(m => "1..0 # SKIP " + m).getOrElse("1.." + plan.plan) + + /** Returns a bail out with a reason. + * + * @example `Bail out! Not supported on this platform.` + */ + def bailOut (message: String): String = + "Bail out! " + message +} diff --git a/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala new file mode 100644 index 0000000..1c2e88d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala @@ -0,0 +1,40 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.Plan + +/** An event emitted while parsing a TAP stream. */ +sealed trait TAPEvent + +/** The start of a TAP stream. */ +case object StartEvent extends TAPEvent + +/** The end of a TAP stream. + * @param result The [[com.iinteractive.test.tap.TAPResult TAPResult]] + * containing information about all of the tests which just + * finished running. This will be the same thing that is + * returned by the call to + * [[com.iinteractive.test.tap.Parser Parser]]'s `parse` + * method. + */ +case class EndEvent private[tap] (result: TAPResult) extends TAPEvent + +/** An individual test result. + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * containing information about the corresponding test. + */ +case class ResultEvent private[tap] (result: TestResult) extends TAPEvent + +/** A test plan. + * @param plan The [[com.iinteractive.test.Plan Plan]] corresponding to the + * line that was parsed. + */ +case class PlanEvent private[tap] (plan: Plan) extends TAPEvent + +/** The start of a subtest (currently unused). */ +case object SubtestStartEvent extends TAPEvent + +/** The end of a subtest (currently unused). */ +case class SubtestEndEvent private[tap] (result: TestResult) extends TAPEvent + +/** A comment (currently unused). */ +case class CommentEvent private[tap] (text: String) extends TAPEvent diff --git a/src/main/scala/com/iinteractive/test/tap/TAPResult.scala b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala new file mode 100644 index 0000000..c3c4926 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala @@ -0,0 +1,94 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** The summarized results of a TAP stream. Contains the + * [[com.iinteractive.test.Plan Plan]] that was given, as well as a list of + * [[com.iinteractive.test.tap.TestResult TestResults]] corresponding to each + * of the tests in the stream. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] from the TAP stream + * @param results The list of + * [[com.iinteractive.test.tap.TestResult TestResults]] from + * the TAP stream + */ +class TAPResult (val plan: Plan, val results: Seq[TestResult]) { + /** Returns true if the number of tests executed was compatible with the + * provided test plan. + */ + val matchesPlan = plan match { + case NumericPlan(n) => results.length == n + case _ => results.length == 0 + } + + /** Returns the number of test failures in the TAP stream. */ + val fails = results.count { r => + !r.passed && !r.directive.isDefined + } + + /** Returns true if all of the tests passed. */ + val testsPassed = fails == 0 + + /** Returns true if the TAP stream overall passed. + * + * Differs from `testsPassed` in that it also takes into account things + * like invalid plans. + */ + val success = plan match { + case SkipAll(_) => true + case _ => results.length > 0 && fails == 0 && matchesPlan + } + + /** Returns the exit code to use if running this test on its own. + * + * Success is indicated by 0, invalid TAP streams (such as incorrect plans) + * by 255, and other kinds of failures by the failure count. + */ + val exitCode = + if (success) { + 0 + } + else if (!matchesPlan || results.length == 0) { + 255 + } + else { + fails + } +} + +/** The result of a single test. + * + * @param passed True if the test passed + * @param number The test number in the TAP stream + * @param description The test description + * @param directive The [[com.iinteractive.test.tap.Directive Directive]] + * (either skip or todo) that was provided for this test, + * if any + * @param subtest The [[com.iinteractive.test.tap.TAPResult]] for the + * subtest that this test corresponds to, if any + */ +class TestResult ( + val passed: Boolean, + val number: Int, + val description: String, + val directive: Option[Directive], + val subtest: Option[TAPResult] +) + +/** A modifier associated with a test result. This is indicated by a `#` at + * the end of the result line, followed by the type of directive, and an + * optional message. + */ +sealed trait Directive { + val message: Option[String] +} + +/** A directive indicating that this test was skipped. */ +case class SkipDirective private[tap] ( + message: Option[String] +) extends Directive + +/** A directive indicating that this test is known to fail. */ +case class TodoDirective private[tap] ( + message: Option[String] +) extends Directive diff --git a/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala new file mode 100644 index 0000000..62fdf02 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala @@ -0,0 +1,196 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test._ + +/** This class provides a convenient yet low level API for generating TAP + * streams. Each instance of this class handles a single TAP stream, and + * keeps track of things like the current test number for you. All + * TAP-producing methods write the TAP lines to `Console.out` or + * `Console.err`, so you can override those (via `Console.withOut` or + * `Console.withErr`). + */ +class TestBuilder private ( + plan: Plan, + indent: String, + terminalInUse: Boolean +) { + plan match { + case NoPlan => () + case p => outLine(Producer.plan(p)) + } + + /** Creates a new builder instance, and emits the corresponding plan line, + * unless the plan is not given. + * + * @param plan [[com.iinteractive.test.Plan plan]] for this test. + * @param terminalInUse Whether this test is being run from a harness which + * will not just be writing directly to the output. + * This will make things written to `Console.err` have + * a newline prepended, so that they always start on + * an empty line. + */ + def this (plan: Plan = NoPlan, terminalInUse: Boolean = false) = + this(plan, "", terminalInUse) + + /** Create a new TestBuilder instance, to be used to run a subtest. This new + * instance will have all of its lines prefixed by an additional level of + * indentation. This instance will still need to have `doneTesting` + * called on it, and the result of the subtest will still need to be + * reported as a separate test result through `ok`. + */ + def cloneForSubtest (newPlan: Plan): TestBuilder = + new TestBuilder(newPlan, indent + " ", terminalInUse) + + /** Reports a single test result to `Console.out`. */ + def ok (test: Boolean) { + state.ok(test) + outLine(Producer.result(test, state.currentTest)) + } + + /** Reports a single test result with description to `Console.out`. */ + def ok (test: Boolean, description: String) { + state.ok(test) + outLine(Producer.result(test, state.currentTest, description)) + } + + /** Reports a single TODO test result to `Console.out`. */ + def todo (todo: String, test: Boolean) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, todo)) + } + + /** Reports a single TODO test result with description to `Console.out`. */ + def todo (todo: String, test: Boolean, description: String) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, description, todo)) + } + + /** Reports a single skipped test result to `Console.out`. */ + def skip (reason: String) { + state.ok(true) + outLine(Producer.skip(state.currentTest, reason)) + } + + /** Writes a comment line to `Console.err`. This will allow it to be + * visible in most summarizing harnesses (which consume and parse + * everything that goes to `Console.out`). + */ + def diag (message: String) { + errLine(Producer.comment(message)) + } + + /** Write a comment line to `Console.out`. This will typically only be + * visible in the raw TAP stream. + */ + def note (message: String) { + outLine(Producer.comment(message)) + } + + /** Abort the current test, with a message. */ + def bailOut (message: String) { + val bailOutMessage = Producer.bailOut(message) + outLine(bailOutMessage) + throw new BailOutException(bailOutMessage) + } + + /** Finalize the current builder instance. This writes the auto-calculated + * plan to `Console.out` if the plan type was `NoPlan` and reports a + * summary of the test results as a comment to `Console.err`. + * + * @return whether or not the test class as a whole passed. + */ + def doneTesting: Boolean = { + plan match { + case NumericPlan(_) => printErrors + case SkipAll(_) => () + case NoPlan => { + outLine(Producer.plan(state.currentTest)) + printErrors + } + } + state.isPassing + } + + /** The exit code to use, in harnesses that run a single test. Passing tests + * return 0, invalid tests (such as running a different number of tests + * than planned) return 255, and all others return the number of failed + * tests. + */ + def exitCode: Int = + if (state.isPassing) { + 0 + } + else if (!state.matchesPlan || state.currentTest == 0) { + 255 + } + else { + state.failCount + } + + private def printErrors { + if (!state.matchesPlan) { + val planCount = (plan match { + case NoPlan => state.currentTest + case p => p.plan + }) + val planned = planCount + " test" + (if (planCount > 1) "s" else "") + val ran = state.currentTest + diag("Looks like you planned " + planned + " but ran " + ran + ".") + } + + if (state.currentTest == 0) { + diag("No tests run!") + } + + if (state.failCount > 0) { + val count = state.failCount + val fails = count + " test" + (if (count > 1) "s" else "") + val total = + state.currentTest + (if (state.matchesPlan) "" else " run") + diag("Looks like you failed " + fails + " of " + total + ".") + } + } + + private val state = new TestState + + private def outLine (str: String) { + Console.out.println(withIndent(str)) + } + + private def errLine (str: String) { + if (terminalInUse) { + Console.err.print("\n") + } + Console.err.println(withIndent(str)) + } + + private def withIndent (str: String): String = + str.split("\n").map(s => indent + s).mkString("\n") + + private class TestState { + var passCount = 0 + var failCount = 0 + + def ok (cond: Boolean) { + if (cond) { + passCount += 1 + } + else { + failCount += 1 + } + } + + def currentTest: Int = + failCount + passCount + + def matchesPlan: Boolean = plan match { + case NumericPlan(p) => p.plan == currentTest + case _ => true + } + + def isPassing: Boolean = plan match { + case SkipAll(_) => true + case _ => currentTest > 0 && failCount == 0 && matchesPlan + } + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/package.scala b/src/main/scala/com/iinteractive/test/tap/package.scala new file mode 100644 index 0000000..312f996 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/package.scala @@ -0,0 +1,9 @@ +package com.iinteractive.test + +/** Classes for TAP generation and parsing. */ +package object tap { + /** Exception representing an error during parsing. It is thrown when a TAP + * line isn't recognized. + */ + case class ParseException (message: String) extends RuntimeException(message) +} |