From 7efb2caf7d8832a7d3a9d2ac55862e43267a3eb2 Mon Sep 17 00:00:00 2001 From: Jesse Luehrs Date: Wed, 6 Mar 2013 15:32:26 -0600 Subject: move the directory structure too --- .../scala/com/iinteractive/test/ExternalTest.scala | 73 +++ src/main/scala/com/iinteractive/test/Test.scala | 34 ++ .../scala/com/iinteractive/test/TestMore.scala | 519 +++++++++++++++++++++ .../test/harness/MultiTestReporter.scala | 16 + .../com/iinteractive/test/harness/Reporter.scala | 15 + .../test/harness/SummarizedTests.scala | 42 ++ .../test/harness/SummaryReporter.scala | 192 ++++++++ .../iinteractive/test/harness/TAPReporter.scala | 12 + .../iinteractive/test/harness/TestHarness.scala | 110 +++++ .../com/iinteractive/test/harness/package.scala | 33 ++ src/main/scala/com/iinteractive/test/package.scala | 74 +++ .../com/iinteractive/test/sbt/Fingerprint.scala | 11 + .../com/iinteractive/test/sbt/Framework.scala | 18 + .../scala/com/iinteractive/test/sbt/Runner.scala | 25 + .../com/iinteractive/test/sbt/SBTReporter.scala | 99 ++++ .../scala/com/iinteractive/test/sbt/package.scala | 4 + .../scala/com/iinteractive/test/tap/Consumer.scala | 131 ++++++ .../scala/com/iinteractive/test/tap/Parser.scala | 232 +++++++++ .../scala/com/iinteractive/test/tap/Producer.scala | 63 +++ .../scala/com/iinteractive/test/tap/TAPEvent.scala | 40 ++ .../com/iinteractive/test/tap/TAPResult.scala | 94 ++++ .../com/iinteractive/test/tap/TestBuilder.scala | 196 ++++++++ .../scala/com/iinteractive/test/tap/package.scala | 9 + src/main/scala/org/perl8/test/ExternalTest.scala | 73 --- src/main/scala/org/perl8/test/Test.scala | 34 -- src/main/scala/org/perl8/test/TestMore.scala | 519 --------------------- .../org/perl8/test/harness/MultiTestReporter.scala | 16 - .../scala/org/perl8/test/harness/Reporter.scala | 15 - .../org/perl8/test/harness/SummarizedTests.scala | 42 -- .../org/perl8/test/harness/SummaryReporter.scala | 192 -------- .../scala/org/perl8/test/harness/TAPReporter.scala | 12 - .../scala/org/perl8/test/harness/TestHarness.scala | 110 ----- .../scala/org/perl8/test/harness/package.scala | 33 -- src/main/scala/org/perl8/test/package.scala | 74 --- .../scala/org/perl8/test/sbt/Fingerprint.scala | 11 - src/main/scala/org/perl8/test/sbt/Framework.scala | 18 - src/main/scala/org/perl8/test/sbt/Runner.scala | 25 - .../scala/org/perl8/test/sbt/SBTReporter.scala | 99 ---- src/main/scala/org/perl8/test/sbt/package.scala | 4 - src/main/scala/org/perl8/test/tap/Consumer.scala | 131 ------ src/main/scala/org/perl8/test/tap/Parser.scala | 232 --------- src/main/scala/org/perl8/test/tap/Producer.scala | 63 --- src/main/scala/org/perl8/test/tap/TAPEvent.scala | 40 -- src/main/scala/org/perl8/test/tap/TAPResult.scala | 94 ---- .../scala/org/perl8/test/tap/TestBuilder.scala | 196 -------- src/main/scala/org/perl8/test/tap/package.scala | 9 - .../com/iinteractive/test/ExtensionTest.scala | 57 +++ .../com/iinteractive/test/ExternalTestTest.scala | 3 + .../scala/com/iinteractive/test/PlanTest.scala | 29 ++ .../scala/com/iinteractive/test/TestMoreTest.scala | 147 ++++++ .../com/iinteractive/test/tap/ParserTest.scala | 127 +++++ .../iinteractive/test/tap/TestBuilderTest.scala | 241 ++++++++++ src/test/scala/org/perl8/test/ExtensionTest.scala | 57 --- .../scala/org/perl8/test/ExternalTestTest.scala | 3 - src/test/scala/org/perl8/test/PlanTest.scala | 29 -- src/test/scala/org/perl8/test/TestMoreTest.scala | 147 ------ src/test/scala/org/perl8/test/tap/ParserTest.scala | 127 ----- .../scala/org/perl8/test/tap/TestBuilderTest.scala | 241 ---------- 58 files changed, 2646 insertions(+), 2646 deletions(-) create mode 100644 src/main/scala/com/iinteractive/test/ExternalTest.scala create mode 100644 src/main/scala/com/iinteractive/test/Test.scala create mode 100644 src/main/scala/com/iinteractive/test/TestMore.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/Reporter.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/TAPReporter.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/TestHarness.scala create mode 100644 src/main/scala/com/iinteractive/test/harness/package.scala create mode 100644 src/main/scala/com/iinteractive/test/package.scala create mode 100644 src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala create mode 100644 src/main/scala/com/iinteractive/test/sbt/Framework.scala create mode 100644 src/main/scala/com/iinteractive/test/sbt/Runner.scala create mode 100644 src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala create mode 100644 src/main/scala/com/iinteractive/test/sbt/package.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/Consumer.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/Parser.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/Producer.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/TAPEvent.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/TAPResult.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/TestBuilder.scala create mode 100644 src/main/scala/com/iinteractive/test/tap/package.scala delete mode 100644 src/main/scala/org/perl8/test/ExternalTest.scala delete mode 100644 src/main/scala/org/perl8/test/Test.scala delete mode 100644 src/main/scala/org/perl8/test/TestMore.scala delete mode 100644 src/main/scala/org/perl8/test/harness/MultiTestReporter.scala delete mode 100644 src/main/scala/org/perl8/test/harness/Reporter.scala delete mode 100644 src/main/scala/org/perl8/test/harness/SummarizedTests.scala delete mode 100644 src/main/scala/org/perl8/test/harness/SummaryReporter.scala delete mode 100644 src/main/scala/org/perl8/test/harness/TAPReporter.scala delete mode 100644 src/main/scala/org/perl8/test/harness/TestHarness.scala delete mode 100644 src/main/scala/org/perl8/test/harness/package.scala delete mode 100644 src/main/scala/org/perl8/test/package.scala delete mode 100644 src/main/scala/org/perl8/test/sbt/Fingerprint.scala delete mode 100644 src/main/scala/org/perl8/test/sbt/Framework.scala delete mode 100644 src/main/scala/org/perl8/test/sbt/Runner.scala delete mode 100644 src/main/scala/org/perl8/test/sbt/SBTReporter.scala delete mode 100644 src/main/scala/org/perl8/test/sbt/package.scala delete mode 100644 src/main/scala/org/perl8/test/tap/Consumer.scala delete mode 100644 src/main/scala/org/perl8/test/tap/Parser.scala delete mode 100644 src/main/scala/org/perl8/test/tap/Producer.scala delete mode 100644 src/main/scala/org/perl8/test/tap/TAPEvent.scala delete mode 100644 src/main/scala/org/perl8/test/tap/TAPResult.scala delete mode 100644 src/main/scala/org/perl8/test/tap/TestBuilder.scala delete mode 100644 src/main/scala/org/perl8/test/tap/package.scala create mode 100644 src/test/scala/com/iinteractive/test/ExtensionTest.scala create mode 100644 src/test/scala/com/iinteractive/test/ExternalTestTest.scala create mode 100644 src/test/scala/com/iinteractive/test/PlanTest.scala create mode 100644 src/test/scala/com/iinteractive/test/TestMoreTest.scala create mode 100644 src/test/scala/com/iinteractive/test/tap/ParserTest.scala create mode 100644 src/test/scala/com/iinteractive/test/tap/TestBuilderTest.scala delete mode 100644 src/test/scala/org/perl8/test/ExtensionTest.scala delete mode 100644 src/test/scala/org/perl8/test/ExternalTestTest.scala delete mode 100644 src/test/scala/org/perl8/test/PlanTest.scala delete mode 100644 src/test/scala/org/perl8/test/TestMoreTest.scala delete mode 100644 src/test/scala/org/perl8/test/tap/ParserTest.scala delete mode 100644 src/test/scala/org/perl8/test/tap/TestBuilderTest.scala diff --git a/src/main/scala/com/iinteractive/test/ExternalTest.scala b/src/main/scala/com/iinteractive/test/ExternalTest.scala new file mode 100644 index 0000000..7855aa8 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/ExternalTest.scala @@ -0,0 +1,73 @@ +package com.iinteractive.test + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.concurrent.Future._ +import scala.annotation.tailrec + +/** Runs an external process which emits TAP, and parses it as a test. + * + * This test class can be used if you have existing tests that you would like + * to be able to include in a test suite using this framework. You just need + * to write a test class for each external test that looks like this: + * + * {{{ + * class MyTest1 extends ExternalTest("perl", "t/basic.t") + * }}} + * + * This will run your external process, and use its TAP stream as its output. + * This will allow it to, for instance, be a part of the test suite that runs + * via `sbt test`. As with any other test class, its stdout and stderr will + * be sent to `Console.out` and `Console.err`, where they can be overridden + * as needed. + */ +class ExternalTest (cmdLine: String*) extends Test { + protected def runTests (raw: Boolean): Int = { + val processBuilder = new ProcessBuilder(cmdLine: _*) + + // Ensure that if stdout and stderr are both pointing to the same place (a + // terminal or file or something) that they remain synchronized. This is + // only possible if the endpoint they are streaming to is the same place + // underneath, with no extra processing in between. (This is safe because + // stdout is typically line-buffered, and we only ever output a line at a + // time when writing TAP, so in theory, buffering of the underlying file + // descriptor shouldn't make a difference here) + if (Console.out eq System.out) { + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT) + } + if (Console.err eq System.err) { + processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT) + } + + val process = processBuilder.start + + val streams = Seq( + Console.out -> process.getInputStream, + Console.err -> process.getErrorStream + ) + + val listeners = streams.map { case (out, in) => + Future { + val buf = new Array[Byte](1024) + + @tailrec + def read { + val bytes = in.read(buf) + if (bytes >= 0) { + out.print(new String(buf.take(bytes))) + read + } + } + + read + true + } + } + + val exitCode = process.waitFor + Await.ready(Future.sequence(listeners), Duration.Inf) + exitCode + } +} diff --git a/src/main/scala/com/iinteractive/test/Test.scala b/src/main/scala/com/iinteractive/test/Test.scala new file mode 100644 index 0000000..2161de3 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/Test.scala @@ -0,0 +1,34 @@ +package com.iinteractive.test + +/** Base trait for test classes in this framework. Any tests that should be + * autodiscovered by `sbt test` should extend this trait, and implement + * [[runTests]]. + */ +trait Test { + /** Runs the test. The TAP stream will be written to Console.out and + * Console.err, so you can swap these out as required in order to parse it. + * + * @return The exit code that the test produced. Success is indicated by 0, + * failure to run the correct number of tests by 255, and any other + * failure by the number of tests that failed. This should be used + * by reporters which run a single test, which can call + * `sys.exit(exitCode)` + */ + def run: Int = + runTests(false) + + /** Runs the test just like [[run]], but in a way that makes sense when test + * results are being summarized rather than directly displayed. + * + * Summarizing test reporters tend to repeatedly update the same line on + * the terminal, so this method makes calls to + * [[com.iinteractive.test.tap.TestBuilder#diag diag]] (which sends + * messages to stderr, where they are typically displayed as-is) prefix the + * message with a newline, to ensure that the output starts on its own + * line. + */ + def runInHarness: Int = + runTests(true) + + protected def runTests (terminalInUse: Boolean): Int +} diff --git a/src/main/scala/com/iinteractive/test/TestMore.scala b/src/main/scala/com/iinteractive/test/TestMore.scala new file mode 100644 index 0000000..eb5210b --- /dev/null +++ b/src/main/scala/com/iinteractive/test/TestMore.scala @@ -0,0 +1,519 @@ +package com.iinteractive.test + +import scala.util.matching.Regex + +import com.iinteractive.test.tap.TestBuilder + +/** This class is an implementation of the excellent + * [[https://metacpan.org/module/Test::More Test::More]] testing library for + * Perl. It provides a simple assertion-based testing API, which produces + * [[http://en.wikipedia.org/wiki/Test_Anything_Protocol TAP]], which can be + * parsed by any TAP consumer. This library includes several TAP-consuming + * harnesses to use with tests using this class, including one that supports + * testing via `sbt test`. + * + * ==Basics== + * + * The most basic test looks like this: + * + * 
+  * class MyTest extends TestMore {
+  *   ok(true)
+  * }
+  *
+ * + * This runs a test containing a single assertion. This will generate a TAP + * stream that looks like this: + * + * 
+  * ok 1
+  * 1..1
+  *
+ * + * which can be parsed by one of the test harnesses provided by this library. + * + * ==Running tests== + * + * The simplest way to run tests is through sbt. You can register this + * framework with sbt by adding this line to your `build.sbt` file: + * + * 
+  * testFrameworks += new TestFramework("com.iinteractive.test.sbt.Framework")
+  *
+ * + * Then, any classes in your test directory which extend `TestMore` will be + * automatically detected and run. + * + * ==Assertions== + * + * This class contains many more assertion methods than just `ok`. Here is a + * more extensive example (borrowed from Test::More's documentation): + * + * 
+  * class MyTest extends TestMore {
+  *   ok(got == expected, testName)
+  *
+  *   is(got, expected, testName)
+  *   isnt(got, expected, testName)
+  *
+  *   diag("here's what went wrong")
+  *
+  *   like(got, """expected""".r, testName)
+  *   unlike(got, """expected""".r, testName)
+  *
+  *   skip(howMany, why) {
+  *     ok(foo(), testName)
+  *     is(foo(42), 23, testName)
+  *   }
+  *
+  *   todo(why) {
+  *     ok(foo(), testName)
+  *     is(foo(42), 23, testName)
+  *   }
+  *
+  *   pass(testName)
+  *   fail(testName)
+  *
+  *   BAIL_OUT(why)
+  * }
+  *
+ * + * The difference between the simple `ok` method and the more specific + * methods like `is` and `like` is in how failures are reported. If you write + * this: + * + * 
+  * ok(1 == 2)
+  *
+ * + * the output will look like this: + * + * 
+  * not ok 1
+  * #   Failed test at MyTest.scala line 4.
+  *
+ * + * On the other hand, a more specific assertion such as: + * + * 
+  * is(1, 2)
+  *
+ * + * will produce more useful output: + * + * 
+  * not ok 1
+  * #   Failed test at MyTest.scala line 4.
+  * #          got: 1
+  * #     expected: 2
+  *
+ * + * In addition to assertions, there are also several methods which take a + * block of code to run, to modify the assertions contained in that block. + * + * The `todo` method runs tests which are expected to fail. If they do fail, + * the failure is reported to the test harness as a normal succeeding test, + * and nothing happens. If they succeed, they are still reported as a + * succeeding test, but a message is displayed to the user indicating that + * the todo indication can be removed. + * + * The `skip` method takes a block which should not be run at all. This is + * similar to `todo`, except that it is useful for tests which could cause + * problems if they were to actually run. Since the tests are never run, it's + * not possible to count how many tests there should be, so this must be + * specified as a parameter. + * + * The `subtest` method runs a block of assertions as though they were an + * entirely separate test, and then reports the result of that test as a + * single assertion in the test that called `subtest`. + * + * ==Test plans== + * + * Normally, you can run any number of assertions within your class body, and + * the framework will assume that if no exceptions were thrown, all of the + * assertions that were meant to be run were actually run. Sometimes, + * however, that may not be a safe assumption, especially with heavily + * callback-driven code. In this case, you can specify exactly how many tests + * you intend to run, and the number of tests actually run will be checked + * against this at the end. To declare this, give a number to the `TestMore` + * constructor: + * + * 
+  * class MyTest extends TestMore(5) {
+  *   ???
+  * }
+  *
+ * + * In addition, if the entire test should be skipped, you can give a plan of + * `SkipAll()`: + * + * 
+  * class MyTest extends TestMore(SkipAll()) {
+  *   ???
+  * }
+  *
+ * + * ==Extensions== + * + * These assertion methods are written with the intention of being + * composable. You can write your own test methods which call `is` or `ok` on + * more specific bits of data. The one issue here is that, as shown above, + * test failure messages refer to the file and line where the `is` or `ok` + * call was made. If you want this to instead point at the line where your + * assertion helper method was called, you can use the `hideTestMethod` + * method like this: + * + * 
+  * trait MyTestHelpers { this: TestMore =>
+  *   def notok (cond: Boolean) = hideTestMethod {
+  *     ok(!cond)
+  *   }
+  * }
+  *
+ * + * This way, the test failure will be reported from the line where `notok` + * was called, not from the call to `ok` in the `notok` method. + */ +class TestMore (plan: Plan = NoPlan) extends Test with DelayedInit { + def delayedInit (body: => Unit) { + testBody = { terminalInUse => + todo = None + builder = new TestBuilder(plan, terminalInUse) + plan match { + case SkipAll(_) => () + case _ => body + } + } + } + + protected def runTests (terminalInUse: Boolean): Int = { + if (testBody == null) { + delayedInit { } + } + + testBody(terminalInUse) + builder.doneTesting + builder.exitCode + } + + /** Assert that a condition is true. + * + * @example `ok(response.isSuccess)` + */ + def ok (cond: Boolean): Boolean = + test(cond) + + /** Assert that a condition is true, and describe the assertion. + * + * @example `ok(response.isSuccess, "the response succeeded")` + */ + def ok (cond: Boolean, desc: String): Boolean = + testWithDesc(cond, desc) + + /** Assert that two objects are equal (using `==`). + * + * @example `is(response.status, 200)` + */ + def is[T] (got: T, expected: T): Boolean = + test(got == expected, isMessage(got, expected)) + + /** Assert that two objects are equal (using `==`), and describe the + * assertion. + * + * @example `is(response.status, 200, "we got a 200 OK response")` + */ + def is[T] (got: T, expected: T, desc: String): Boolean = + testWithDesc(got == expected, desc, isMessage(got, expected)) + + /** Assert that two objects are not equal (using `!=`). + * + * @example `isnt(response.body, "")` + */ + def isnt[T] (got: T, expected: T): Boolean = + test(got != expected, isntMessage(got)) + + /** Assert that two objects are not equal (using `!=`), and describe the + * assertion. + * + * @example `isnt(response.body, "", "we got a response body")` + */ + def isnt[T] (got: T, expected: T, desc: String): Boolean = + testWithDesc(got != expected, desc, isntMessage(got)) + + /** Assert that a string matches a regular expression. + * + * @example `like(response.header("Content-Type"), """text/x?html""".r)` + */ + def like (got: String, rx: Regex): Boolean = + test(rx.findFirstIn(got).nonEmpty, likeMessage(got, rx)) + + /** Assert that a string matches a regular expression, and describe the + * assertion. + * + * @example `like(response.header("Content-Type"), """text/x?html""".r, "we got an html content type")` + */ + def like (got: String, rx: Regex, desc: String): Boolean = + testWithDesc(rx.findFirstIn(got).nonEmpty, desc, likeMessage(got, rx)) + + /** Assert that a string doesn't match a regular expression. + * + * @example `unlike(response.header("Authorization"), """^Digest.*""".r)` + */ + def unlike (got: String, rx: Regex): Boolean = + test(rx.findFirstIn(got).isEmpty, unlikeMessage(got, rx)) + + /** Assert that a string doesn't match a regular expression. + * + * @example `unlike(response.header("Authorization"), """^Digest.*""".r, "we don't support digest authentication")` + */ + def unlike (got: String, rx: Regex, desc: String): Boolean = + testWithDesc(rx.findFirstIn(got).isEmpty, desc, unlikeMessage(got, rx)) + + /** An assertion that always succeeds. + * + * @example `pass()` + */ + def pass: Boolean = + ok(true) + + /** An assertion that always succeeds, with a reason. + * + * @example `pass("this line of code should be executed")` + */ + def pass (desc: String): Boolean = + ok(true, desc) + + /** An assertion that always fails. + * + * @example `fail()` + */ + def fail: Boolean = + ok(false) + + /** An assertion that always fails, with a reason. + * + * @example `fail("we should never get here")` + */ + def fail (desc: String): Boolean = + ok(false, desc) + + /** Output a comment to `Console.err`. This is intended to be visible to + * users even when running the test under a summarizing harness. + * + * @example `diag("Testing with Scala " + util.Properties.versionString)` + */ + def diag (message: String) { + builder.diag(message) + } + + /** Output a comment to `Console.out`. This is intended to only be visible + * when viewing the raw TAP stream. + * + * @example `note("Starting the response tests")` + */ + def note (message: String) { + builder.note(message) + } + + /** Halt execution of the entire test suite. + * + * @example `BAIL_OUT("can't connect to the database!")` + */ + def BAIL_OUT (desc: String) { + builder.bailOut(desc) + } + + /** Mark a block of tests as expected to fail. If the tests which run in the + * todo block fail, they will not be treated as test failures, and if they + * succeed, the user will be notified. + * + * @example `todo("waiting on fixes elsewhere") { ??? }` + */ + def todo (reason: String)(body: => Unit) { + val oldTodo = todo + try { + todo = Some(reason) + body + } + finally { + todo = oldTodo + } + } + + /** Mark a block of tests that should not be run at all. They are treated as + * always passing. + * + * @example `skip(3, "too dangerous to run for now") { ??? }` + */ + def skip (count: Int, reason: String)(body: => Unit) { + for (i <- 1 to count) { + builder.skip(reason) + } + } + + /** Declare a logical group of assertions, to be run as a single test. This + * is effectively an entirely separate test, which is run, and the result + * of that test is reported as a single assertion in the test that contains + * it. The subtest can specify its own plan in the same way that the + * overall test is allowed to. The name will be used as the description for + * the single assertion that the overall test sees. + * + * @example `subtest("response tests") { ??? }` + */ + def subtest ( + name: String, + plan: Plan = NoPlan + )(body: => Unit): Boolean = { + val oldBuilder = builder + val success = try { + builder = oldBuilder.cloneForSubtest(plan) + body + builder.doneTesting + } + finally { + builder = oldBuilder + } + ok(success, name) + } + + /** A helper method which should be used to wrap test utility methods. + * Normally, when tests fail, a message is printed giving the file and line + * number of the call to the test method. If you write your own test + * methods, they will typically use the existing methods to generate + * assertions, and so the file and line numbers will likely be much less + * useful. Wrapping the body of your method in this method will ensure that + * the file and line number that is reported is the line where your helper + * method is called instead. + * + * @example `def testFixtures = hideTestMethod { ??? }` + */ + def hideTestMethod[T] (body: => T): T = { + // this just adds a method call with a known name to the stack trace, so + // that we can detect it later + body + } + + private def isMessage[T] (got: T, expected: T): String = + " got: '" + got + "'\n" + + " expected: '" + expected + "'\n" + + private def isntMessage[T] (got: T): String = + " got: '" + got + "'\n" + + " expected: anything else\n" + + private def likeMessage (got: String, rx: Regex): String = + " '" + got + "'\n" + + " doesn't match '" + rx + "'\n" + + private def unlikeMessage (got: String, rx: Regex): String = + " '" + got + "'\n" + + " matches '" + rx + "'\n" + + private def testWithDesc ( + cond: Boolean, + desc: String + ): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond, "- " + desc) + case None => builder.ok(cond, "- " + desc) + } + if (!cond) { + failed(Some(desc), None) + } + cond + } + + private def testWithDesc ( + cond: Boolean, + desc: String, + reason: => String + ): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond, "- " + desc) + case None => builder.ok(cond, "- " + desc) + } + if (!cond) { + failed(Some(desc), Some(reason)) + } + cond + } + + private def test (cond: Boolean): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond) + case None => builder.ok(cond) + } + if (!cond) { + failed(None, None) + } + cond + } + + private def test (cond: Boolean, reason: => String): Boolean = { + todo match { + case Some(t) => builder.todo(t, cond) + case None => builder.ok(cond) + } + if (!cond) { + failed(None, Some(reason)) + } + cond + } + + private def failed (desc: Option[String], reason: Option[String]) { + val stack = Thread.currentThread.getStackTrace.drop(1).filter { frame => + !ignoreFrame(frame) + } + val idx = stack.lastIndexWhere { frame => + frame.getClassName == "com.iinteractive.test.TestMore" && + frame.getMethodName == "hideTestMethod" + } + val caller = idx match { + case -1 => stack.headOption + // one level to jump out of hideTestMethod and one level to jump out of + // the method that called hideTestMethod + case i => stack.drop(i + 2).headOption + + } + val (file, line) = caller match { + case Some(frame) => (frame.getFileName, frame.getLineNumber) + case None => ("", "") + } + val message = " " + (todo match { + case Some(_) => "Failed (TODO) test" + case None => "Failed test" + }) + (desc match { + case Some(m) => " '" + m + "'\n " + case None => " " + }) + val trace = "at " + file + " line " + line + "." + val explanation = message + trace + reason.map("\n" + _).getOrElse("") + if (todo.isDefined) { + builder.note(explanation) + } + else { + builder.diag(explanation) + } + } + + protected def ignoreFrame (frame: StackTraceElement): Boolean = { + val className = frame.getClassName + val methodName = frame.getMethodName + + // ignore everything in this class, except the hideTestMethod call which we + // use as a stack trace marker + (className == "com.iinteractive.test.TestMore" && + methodName != "hideTestMethod") || + // when you call a method in a class when the method is defined in a + // trait, it calls a stub which calls the real definition in the trait. + // the trait is represented under the hood as a class with the same name + // as the trait, except with $class appended. this is a gross reliance on + // implementation details that could change at any moment, but i don't + // really see any better options. + """\$class$""".r.findFirstIn(className).nonEmpty + } + + private var todo: Option[String] = _ + private var builder: TestBuilder = _ + private var testBody: Boolean => Unit = _ +} diff --git a/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala new file mode 100644 index 0000000..56f32fd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/MultiTestReporter.scala @@ -0,0 +1,16 @@ +package com.iinteractive.test.harness + +/** Classes that implement `MultiTestReporter` are capable of running a group + * of test classes, given their names. This typically involves some sort of + * summarization. + * + * @see [[com.iinteractive.test.harness.Reporter Reporter]]. + */ +trait MultiTestReporter { + /** Runs the test classes identifed by the list of fully qualified class + * names `testNames`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testNames: Seq[String]): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/Reporter.scala b/src/main/scala/com/iinteractive/test/harness/Reporter.scala new file mode 100644 index 0000000..a47444b --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/Reporter.scala @@ -0,0 +1,15 @@ +package com.iinteractive.test.harness + +/** Classes that implement `Reporter` are capable of running a test class, + * given its name. + * + * @see [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]]. + */ +trait Reporter { + /** Runs the test class identifed by the fully qualified class name + * `testName`. + * + * @return The exit code for the harness to use. Will be 0 on success. + */ + def run (testName: String): Int +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala new file mode 100644 index 0000000..cd555dd --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummarizedTests.scala @@ -0,0 +1,42 @@ +package com.iinteractive.test.harness + +import java.io.{PipedInputStream,PipedOutputStream} +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +import com.iinteractive.test.tap.{Parser,TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.Test + +/** This is a trait for classes that run tests and summarize the results. It + * provides a single `runOneTest` method, which runs a test class and + * produces a stream of [[com.iinteractive.test.tap.TAPEvent TAP events]] + * which can be used to produce whatever summarized output you need. + */ +trait SummarizedTests { + /** Runs a single [[com.iinteractive.test.Test test]] instance, calling `cb` + * with each [[com.iinteractive.test.tap.TAPEvent TAP event]] as it is + * produced. + * + * @return The overall result of the test instance. + */ + protected def runOneTest (test: Test, cb: TAPEvent => Unit): TAPResult = { + val out = new PipedOutputStream + val in = new PipedInputStream(out) + + val testFuture = Future { + Console.withOut(out) { + test.runInHarness + } + out.close + } + + val parser = new Parser(cb) + val result = parser.parse(in) + in.close + Await.ready(testFuture, Duration.Inf) + + result + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala new file mode 100644 index 0000000..a5fe1e0 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/SummaryReporter.scala @@ -0,0 +1,192 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap.{TAPEvent,TAPResult,TodoDirective} +import com.iinteractive.test.tap.{StartEvent,ResultEvent,PlanEvent,EndEvent} +import com.iinteractive.test.Test + +/** Runs a series of tests. The TAP output from these tests is parsed, and + * output is produced which is similar in style to Perl's + * [[https://metacpan.org/module/Test::Harness Test::Harness]]. + */ +class SummaryReporter extends MultiTestReporter with SummarizedTests { + def run (testNames: Seq[String]): Int = { + val results = runTests(testNames) + val success = results.values.forall(_.success) + printTestSummary(success, results) + if (success) 0 else 1 + } + + protected def runTests (testNames: Seq[String]): Map[String, TAPResult] = { + val maxLength = testNames.map(_.length).max + + testNames.map { name => + val callbackGenerator: () => TAPEvent => Unit = () => { + var width = 0 + var tests = 0 + var plan: Option[Int] = None + + def status = { + tests + "/" + plan.getOrElse("?") + } + + def printStatus (st: String) { + print("\r" + (" " * width) + "\r") + val line = + name + " " + ("." * (maxLength - name.length)) + ".. " + st + width = line.length + print(line) + Console.out.flush + } + + (e: TAPEvent) => e match { + case StartEvent => { + printStatus("") + } + case PlanEvent(p) => { + plan = Some(p.plan) + printStatus(status) + } + case ResultEvent(r) => { + tests += 1 + printStatus(status) + } + case EndEvent(result) => { + if (result.success) { + printStatus("") + println("ok") + } + else { + val results = result.results.length + val failed = result.results.count { t => + !t.passed && !t.directive.isDefined + } + + printStatus("") + println("Dubious, test returned " + result.exitCode) + println("Failed " + failed + "/" + results + " subtests") + } + } + case _ => () + } + } + + name -> runOneTest(newInstance[Test](name), callbackGenerator()) + }.toMap + } + + protected def printTestSummary ( + success: Boolean, + results: Map[String, TAPResult] + ) { + printSuccess(success) + printLongSummary(results) + printShortSummary(results) + printPassFail(success, results) + } + + private def printSuccess (success: Boolean) { + if (success) { + println("All tests successful.") + } + } + + private def printShortSummary (results: Map[String, TAPResult]) { + val files = results.size + val tests = results.values.map(_.results.length).sum + println("Files=" + files + ", Tests=" + tests) + } + + private def printLongSummary (results: Map[String, TAPResult]) { + val todoSucceeded = results.mapValues { r => + r.results.filter { t => + t.directive match { + case Some(TodoDirective(_)) => t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testsFailed = results.mapValues { r => + r.results.filter { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_._2.length > 0) + + val testNames = (todoSucceeded ++ testsFailed).keys + + if (testNames.nonEmpty) { + println("") + println("Test Summary Report") + println("-------------------") + + val maxLength = testNames.map(_.length).max + + for (name <- testNames) { + val result = results(name) + + println( + name + (" " * (maxLength - name.length)) + " " + + "(Tests: " + result.results.length + " " + + "Failed: " + testsFailed.getOrElse(name, Nil).length + ")" + ) + + if (testsFailed.isDefinedAt(name)) { + val fails = testsFailed(name) + println( + " Failed test" + (if (fails.length > 1) "s" else "") + ": " + + fails.map(_.number).mkString(", ") + ) + } + + if (todoSucceeded.isDefinedAt(name)) { + val todos = todoSucceeded(name) + println( + " TODO passed: " + + todos.map(_.number).mkString(", ") + ) + } + + val exitCode = results(name).exitCode + if (exitCode != 0) { + println(" Non-zero exit status: " + exitCode) + } + } + } + } + + private def printPassFail ( + success: Boolean, + results: Map[String, TAPResult] + ) { + if (success) { + println("Result: PASS") + } + else { + println("Result: FAIL") + + val testResults = results.values + + val testsFailed = testResults.map { r => + r.results.count { t => + t.directive match { + case None => !t.passed + case _ => false + } + } + }.filter(_ > 0) + val failedFiles = testsFailed.size + val failedTests = testsFailed.sum + + val allFiles = testResults.size + val allTests = testResults.map(_.results.length).sum + + println( + "Failed " + failedFiles + "/" + allFiles + " test programs. " + + failedTests + "/" + allTests + " subtests failed." + ) + } + } +} diff --git a/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala new file mode 100644 index 0000000..8a4dc9f --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TAPReporter.scala @@ -0,0 +1,12 @@ +package com.iinteractive.test.harness + +import com.iinteractive.test.tap +import com.iinteractive.test.Test + +/** Runs a single test. The TAP stream from that test is written directly to + * stdout/stderr. + */ +class TAPReporter extends Reporter { + def run (testName: String): Int = + newInstance[Test](testName).run +} diff --git a/src/main/scala/com/iinteractive/test/harness/TestHarness.scala b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala new file mode 100644 index 0000000..7b8f4ae --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/TestHarness.scala @@ -0,0 +1,110 @@ +package com.iinteractive.test.harness + +/** This is the entry point to running tests written with this library from + * the command line. Note that this library also implements the + * [[https://github.com/harrah/test-interface common testing interface]] for + * test libraries, so tests should also just work with `sbt test`. + * + * If this application is run and given just a single test class name, it + * will run that test and write its TAP stream to the console. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest + * ok 1 + * ok 2 + * 1..2 + * }}} + * + * If this application is run and given multiple test class names, it will + * run each of those tests, and present a summary report, similar to the one + * produces by + * [[https://metacpan.org/module/Test::Harness Perl's Test::Harness]]. + * + * {{{ + * $ scala com.iinteractive.test.harness.TestHarness MyTest1 MyTest2 + * MyTest1 .. ok + * MyTest2 .. ok + * All tests successful. + * Files=2, Tests=4 + * Result: PASS + * }}} + * + * This application also accepts a few command line options to customize its + * behavior: + * + * - `-r`: Alternative [[com.iinteractive.test.harness.Reporter Reporter]] + * class to use for running a single test. + * - `-R`: Alternative + * [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]] + * class to use for running a group of tests. Also enables using the + * MultiTestReporter for a single test. + * - `--help`: Prints usage information. + */ +object TestHarness { + import com.iinteractive.test.Test + + /** Entry point for the harness application. */ + def main (args: Array[String]) { + val opts = parseOpts(args.toList) + val single = opts("prefer-single").asInstanceOf[Boolean] + + val exitCode = if (single) { + val reporterName = opts("single-reporter").asInstanceOf[String] + val testName = opts("test-classes").asInstanceOf[List[String]].apply(0) + val reporter = newInstance[Reporter](reporterName) + reporter.run(testName) + } + else { + val reporterName = opts("multi-reporter").asInstanceOf[String] + val testNames = opts("test-classes").asInstanceOf[List[String]] + val reporter = newInstance[MultiTestReporter](reporterName) + reporter.run(testNames) + } + + sys.exit(exitCode) + } + + protected def parseOpts (args: List[String]): Map[String, Any] = args match { + case Nil => Map( + "single-reporter" -> "com.iinteractive.test.harness.TAPReporter", + "multi-reporter" -> "com.iinteractive.test.harness.SummaryReporter", + "prefer-single" -> true, + "test-classes" -> Nil + ) + + case "-r" :: singleReporter :: rest => + parseOpts(rest) + ("single-reporter" -> singleReporter) + + case "-R" :: multiReporter :: rest => + parseOpts(rest) ++ Map( + "multi-reporter" -> multiReporter, + "prefer-single" -> false + ) + + case "--help" :: rest => + usage(0) + + case `unknownOption` :: rest => + usage(1) + + case testClass :: rest => { + val opts = parseOpts(rest) + val tests = opts("test-classes").asInstanceOf[List[String]] + opts ++ Map( + "test-classes" -> (testClass :: tests), + "prefer-single" -> tests.isEmpty + ) + } + } + + protected def usage (exitCode: Int) = { + val out = if (exitCode == 0) Console.out else Console.err + out.println("harness [-r ]\n" + + " [-R ]\n" + + " [--help]\n" + + " [...]\n") + sys.exit(exitCode) + } + + private val unknownOption = """^-.*""".r +} diff --git a/src/main/scala/com/iinteractive/test/harness/package.scala b/src/main/scala/com/iinteractive/test/harness/package.scala new file mode 100644 index 0000000..1b74f9d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/harness/package.scala @@ -0,0 +1,33 @@ +package com.iinteractive.test + +/** Classes to handle running test instances and providing output. */ +package object harness { + import scala.reflect.{ClassTag,classTag} + + /** Loads `className`, returning the + * [[http://docs.oracle.com/javase/7/docs/api/java/lang/Class.html Class]] + * instance. + */ + def loadClass[T: ClassTag] (className: String): Class[_] = + classTag[T].runtimeClass.getClassLoader.loadClass(className) + + /** Loads `className` and creates a new instance of it, using the + * no-argument constructor. + */ + def newInstance[T: ClassTag] (className: String): T = + loadClass[T](className).newInstance.asInstanceOf[T] + + /** Loads `className` and creates a new instance of it, using a + * one-argument constructor. Passes `arg` as the argument to the + * constructor. + */ + def newInstance[T: ClassTag, U <: AnyRef: ClassTag] ( + className: String, + arg: U + ): T = { + val classObj = loadClass[T](className) + val argClassObj = classTag[U].runtimeClass + val constructor = classObj.getConstructor(argClassObj) + constructor.newInstance(arg).asInstanceOf[T] + } +} diff --git a/src/main/scala/com/iinteractive/test/package.scala b/src/main/scala/com/iinteractive/test/package.scala new file mode 100644 index 0000000..df077f2 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/package.scala @@ -0,0 +1,74 @@ +package com.iinteractive + +/** This library implements several components of Perl's testing ecosystem in + * Scala. The most useful place to get started to use this library is likely + * [[com.iinteractive.test.TestMore TestMore]]. + */ +package object test { + import language.implicitConversions + + /** Converts an + * [[http://www.scala-lang.org/api/current/index.html#scala.Int Int]] to a + * [[NumericPlan]]. + */ + implicit def intToPlan (p: Int): Plan = + NumericPlan(p) + + /** A test plan. This represents the TAP statement telling how many tests + * will be run. + */ + sealed trait Plan { + /** How many tests will be run. */ + val plan: Int + /** Whether this test was skipped. It should contain `Some(message)` if + * the test is skipped, and `None` otherwise. + */ + val skipAll: Option[String] + } + + /** An explicit plan number. Corresponds to `1..5` in TAP. */ + case class NumericPlan (plan: Int) extends Plan { + /** @inheritdoc + * + * Always `None` for this class. + */ + val skipAll = None + } + + /** A test which did not declare a plan yet. */ + case object NoPlan extends Plan { + /** @inheritdoc + * + * Always 0 for this class. + */ + val plan = 0 + /** @inheritdoc + * + * Always `None` for this class. + */ + val skipAll = None + } + + /** A test which has declared that the entire test has been skipped. + * Corresponds to `1..0 # SKIP [message]` in TAP. + */ + case class SkipAll (message: String) extends Plan { + /** @inheritdoc + * + * Always 0 for this class. + */ + val plan = 0 + /** @inheritdoc + * + * Never `None` for this class. + */ + val skipAll = Some(message) + } + + /** Exception thrown when a test bails out. Corresponds to + * `Bail out! [message]` in TAP. + */ + case class BailOutException ( + message: String + ) extends RuntimeException(message) +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala b/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala new file mode 100644 index 0000000..bab13c5 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Fingerprint.scala @@ -0,0 +1,11 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Fingerprint.java org.scalatools.testing.Fingerprint]]. + */ +object Fingerprint extends testing.SubclassFingerprint { + def isModule: Boolean = false + def superClassName: String = "com.iinteractive.test.Test" +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Framework.scala b/src/main/scala/com/iinteractive/test/sbt/Framework.scala new file mode 100644 index 0000000..bb3d0bb --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Framework.scala @@ -0,0 +1,18 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Framework.java org.scalatools.testing.Framework]]. + */ +class Framework extends testing.Framework { + val name: String = "Perl8 Test" + val tests: Array[testing.Fingerprint] = Array(Fingerprint) + + def testRunner ( + testClassLoader: ClassLoader, + loggers: Array[testing.Logger] + ): testing.Runner = { + new Runner(testClassLoader, loggers) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/Runner.scala b/src/main/scala/com/iinteractive/test/sbt/Runner.scala new file mode 100644 index 0000000..0eee4cf --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/Runner.scala @@ -0,0 +1,25 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +import com.iinteractive.test.harness.SummaryReporter +import com.iinteractive.test.Test + +/** Implementation of + * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Runner2.java org.scalatools.testing.Runner2]] + * using [[com.iinteractive.test.sbt.SBTReporter SBTReporter]]. + */ +class Runner ( + loader: ClassLoader, + loggers: Array[testing.Logger] +) extends testing.Runner2 { + def run ( + testClassName: String, + fingerprint: testing.Fingerprint, + eventHandler: testing.EventHandler, + args: Array[String] + ) { + val reporter = new SBTReporter(loader, loggers, eventHandler) + reporter.run(testClassName) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala b/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala new file mode 100644 index 0000000..34df60d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/SBTReporter.scala @@ -0,0 +1,99 @@ +package com.iinteractive.test.sbt + +import org.scalatools.testing + +import com.iinteractive.test.harness.{Reporter,SummarizedTests} +import com.iinteractive.test.tap.{TAPEvent,ResultEvent,EndEvent} +import com.iinteractive.test.Test + +/** Runs a single test under the SBT test harness. */ +class SBTReporter ( + loader: ClassLoader, + loggers: Array[testing.Logger], + eventHandler: testing.EventHandler +) extends Reporter with SummarizedTests { + def run (testName: String): Int = { + val cb = (e: TAPEvent) => e match { + case ResultEvent(r) => { + val event = new testing.Event { + val testName: String = r.description + val description: String = r.description + val result: testing.Result = + if (r.passed) { + testing.Result.Success + } + else if (r.directive.isDefined) { + testing.Result.Skipped + } + else { + testing.Result.Failure + } + val error: Throwable = null + } + eventHandler.handle(event) + } + case EndEvent(result) => { + val testsPassed = result.success + val correctCode = result.exitCode == 0 + val event = new testing.Event { + val testName: String = "exit code is 0" + val description: String = "exit code is 0" + val result: testing.Result = + if (correctCode) { + testing.Result.Success + } + else { + testing.Result.Failure + } + val error: Throwable = null + } + eventHandler.handle(event) + + if (testsPassed && correctCode) { + logInfo("PASS " + testName) + } + else { + val results = result.results.length + val failed = result.results.count { t => + !t.passed && !t.directive.isDefined + } + + val errors = Seq( + (if (testsPassed) + None + else + Some("failed " + failed + "/" + results)), + (if (correctCode) + None + else + Some("non-zero exit code: " + result.exitCode)) + ).flatten.mkString("(", ", ", ")") + + logError("FAIL " + testName + " " + errors) + } + } + case _ => () + } + + runOneTest( + loader.loadClass(testName).newInstance.asInstanceOf[Test], + cb + ).exitCode + } + + private def logDebug (msg: String) { + loggers.foreach(_.debug(msg)) + } + + private def logInfo (msg: String) { + loggers.foreach(_.info(msg)) + } + + private def logWarn (msg: String) { + loggers.foreach(_.warn(msg)) + } + + private def logError (msg: String) { + loggers.foreach(_.error(msg)) + } +} diff --git a/src/main/scala/com/iinteractive/test/sbt/package.scala b/src/main/scala/com/iinteractive/test/sbt/package.scala new file mode 100644 index 0000000..eeb1f68 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/sbt/package.scala @@ -0,0 +1,4 @@ +package com.iinteractive.test + +/** Classes for interoperating with `sbt test`. */ +package object sbt diff --git a/src/main/scala/com/iinteractive/test/tap/Consumer.scala b/src/main/scala/com/iinteractive/test/tap/Consumer.scala new file mode 100644 index 0000000..09a2f0a --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Consumer.scala @@ -0,0 +1,131 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** Contains a method to parse an individual line of TAP. */ +object Consumer { + /** Parses a line of TAP. + * + * @return A [[com.iinteractive.test.tap.Consumer.Line Line]] object + * corresponding to the parsed line. + */ + def parseLine (line: String): Line = { + commentRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, text) => new CommentLine(text, indent) + } + }.getOrElse { + planRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, p, null) => + new PlanLine(NumericPlan(p.toInt), indent) + case Seq(indent, _, skip) => + new PlanLine(SkipAll(skip), indent) + } + }.getOrElse { + resultRx.findFirstMatchIn(line).map { m => + val indent = m.group(1) + val passed = m.group(2) == null + val number = m.group(3).toInt + val description = m.group(4) match { + case null => "" + case s => s.trim + } + val directive = (m.group(5), m.group(6)) match { + case (null, null) => None + case (d, r) => { + val reason = if (r == null) "" else r + """(?i:skip)""".r.findFirstIn(d) match { + case Some(_) => Some(new SkipDirective(Some(reason))) + case None => Some(new TodoDirective(Some(reason))) + } + } + } + val result = new TestResult( + passed, + number, + description, + directive, + None + ) + new ResultLine(result, indent) + }.getOrElse { + throw ParseException("Couldn't parse line: " + line) + } + } + } + } + + /** The representation of a parsed line of TAP. */ + sealed trait Line { + /** The meaningful portion of the TAP line. */ + def contents: String + /** The indentation of the TAP line (used in subtests). */ + def indent: String + + /** The line itself that was parsed. */ + override def toString: String = + indent + contents + } + + /** A parsed TAP line containing a comment. + * + * @param text The text of the comment (not including the `#`). + */ + case class CommentLine private[Consumer] ( + text: String, + indent: String + ) extends Line { + def contents = "# " + text + } + + /** A parsed TAP line containing a test plan. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] that this line + * represents. + */ + case class PlanLine private[Consumer] ( + plan: Plan, + indent: String + ) extends Line { + def contents = { + val count = plan.plan + val comment = plan match { + case SkipAll(m) => " # SKIP " + m + case _ => "" + } + indent + "1.." + count + comment + } + } + + /** A parsed TAP line containing a test result. + * + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * that this line represents. + */ + case class ResultLine private[Consumer] ( + result: TestResult, + indent: String + ) extends Line { + def contents = { + val success = (if (result.passed) "ok" else "not ok") + " " + val number = result.number + " " + val description = result.description match { + case "" => "" + case s => s + " " + } + val directive = result.directive.map { d => + d match { + case TodoDirective(m) => "# TODO " + m + case SkipDirective(m) => "# skip " + m + } + }.getOrElse("") + indent + success + number + description + directive + } + } + + private val commentRx = """^(\s*)#\s*(.*)""".r + private val planRx = """^(\s*)1..(\d+)\s*(?:# SKIP (.*))?""".r + private val resultRx = + """^(\s*)(not )?ok (\d+)\s*([^#]+)?(?:#\s*(?i:(skip|todo))\s+(.*))?""".r +} diff --git a/src/main/scala/com/iinteractive/test/tap/Parser.scala b/src/main/scala/com/iinteractive/test/tap/Parser.scala new file mode 100644 index 0000000..7bc44d3 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Parser.scala @@ -0,0 +1,232 @@ +package com.iinteractive.test.tap + +import java.io.{ByteArrayInputStream,InputStream,OutputStream} +import scala.annotation.tailrec +import scala.io.Source +import scala.util.parsing.combinator._ +import scala.util.parsing.input.{Position,Reader} + +import com.iinteractive.test.Plan +import com.iinteractive.test.tap.Consumer._ + +/** This class parses a TAP stream. It can either parse it all at once (from a + * string), or it can be used as a streaming parser, where TAP events are + * emitted through a given callback. + */ +class Parser private ( + cb: TAPEvent => Unit, + indent: String +) { + /** Creates a parser instance. + * @param cb The event handler callback. It will be called after each + * meaningful line of TAP, with a + * [[com.iinteractive.test.tap.TAPEvent TAPEvent]] instance + * representing the event that was just parsed. + */ + def this (cb: TAPEvent => Unit = e => ()) = + this(cb, "") + + private def this (indent: String) = + this(e => (), indent) + + /** Parses TAP from an input stream. This variant will actually parse lines + * as they are available to read from the input stream, so this can be used + * as a streaming parser. + */ + def parse (input: InputStream): TAPResult = { + import parser._ + + cb(StartEvent) + tap(new LineReader(input)) match { + case Success(result, _) => { + cb(EndEvent(result)) + result + } + case failure: NoSuccess => throw new ParseException(failure.msg) + } + } + + /** Parses TAP contained in a string. This isn't useful for incremental + * parsing, because the entire input string must be created before + * parsing can begin. + */ + def parse (input: String): TAPResult = + parse(new ByteArrayInputStream(input.getBytes)) + + /** Parses TAP from an output stream. + * + * @todo Doesn't currently work as a streaming parser, since it just + * collects the entire output as a string and feeds it to the parser + * for strings. This could likely be improved. + */ + def parse (input: OutputStream): TAPResult = + parse(input.toString) + + private val parser = new TAPParser(cb, indent) + + private class TAPParser ( + cb: TAPEvent => Unit, + indent: String + ) extends Parsers { + type Elem = Line + + def tap: Parser[TAPResult] = + planFirst | planLast + + private def planFirst: Parser[TAPResult] = + plan ~ rep(result) ^^ { case plan ~ results => + new TAPResult(plan, results) + } + + private def planLast: Parser[TAPResult] = + rep(result) ~ plan ^^ { case results ~ plan => + new TAPResult(plan, results) + } + + private def plan: Parser[Plan] = + planLine ^^ { p => + cb(PlanEvent(p.plan)) + p.plan + } + + private def result: Parser[TestResult] = + simpleResult | subtestResult + + private def simpleResult: Parser[TestResult] = + resultLine ^^ { r => + cb(ResultEvent(r.result)) + r.result + } + + private def subtestResult: Parser[TestResult] = + subtest ~ simpleResult ^^ { case subtest ~ simpleResult => + new TestResult( + simpleResult.passed, + simpleResult.number, + simpleResult.description, + simpleResult.directive, + Some(subtest) + ) + } + + private def subtest: Parser[TAPResult] = + LineParser("subtest") { in => + // can't just return the result directly, because it's of a different + // type (the path dependent type associated with the new Parser + // instance we create here, rather than the path dependent type + // associated with this) + val subParser = new TAPParser( + e => (), + in.first.indent + ) + subParser.tap(in) match { + case subParser.Success(p, rest) => Success(p, rest) + case subParser.Failure(m, rest) => Failure(m, rest) + case subParser.Error(m, rest) => Error(m, rest) + } + } + + private def planLine: Parser[PlanLine] = LineParser("plan") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: PlanLine => + Success(p, in.rest) + case _ => + Failure("Plan line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Plan line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def resultLine: Parser[ResultLine] = LineParser("result") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: ResultLine => + Success(p, in.rest) + case _ => + Failure("Result line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Result line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def LineParser[T] (lineType: String)( + body: Input => ParseResult[T] + ): Parser[T] = Parser { in => + if (in.atEnd) { + Failure(lineType + " line expected, but end of input found", in) + } + else { + body(in) + } + } + } + + private class LineReader ( + in: Iterator[Char], + lineNum: Int + ) extends Reader[Line] { + def this (in: InputStream) = + this(Source.fromInputStream(in), 1) + + def atEnd: Boolean = + nextLine.isEmpty + + def first: Line = + nextLine.getOrElse(throw new RuntimeException("read from empty input")) + + lazy val pos = + new LinePosition(lineNum, nextLine.map(_.toString).getOrElse("")) + + lazy val rest: Reader[Line] = + new LineReader(remainingStream, lineNum + 1) + + private def nextLine: Option[Line] = + state._1 + + private def remainingStream: Iterator[Char] = + state._2 + + private lazy val state: (Option[Line], Iterator[Char]) = + readNextLine(in) + + @tailrec + private def readNextLine ( + stream: Iterator[Char] + ): (Option[Line], Iterator[Char]) = { + if (stream.hasNext) { + val (line, rest) = stream.span(_ != '\n') match { + case (l, r) => (parseLine(l.mkString), r.drop(1)) + } + line match { + case _: CommentLine => readNextLine(rest) + case other => (Some(other), rest) + } + } + else { + (None, in) + } + } + } + + private case class LinePosition ( + line: Int, + lineContents: String + ) extends Position { + def column: Int = 1 + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/Producer.scala b/src/main/scala/com/iinteractive/test/tap/Producer.scala new file mode 100644 index 0000000..3ed8d7e --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Producer.scala @@ -0,0 +1,63 @@ +package com.iinteractive.test.tap + +/** Contains functions for producing individual lines of TAP. */ +object Producer { + import com.iinteractive.test.Plan + + /** Returns a test result. + * + * @example `ok 4` + */ + def result (cond: Boolean, num: Int): String = + (if (cond) "ok " else "not ok ") + num + + /** Returns a test result that contains a description. + * + * @example `ok 28 - our test succeeded` + */ + def result (cond: Boolean, num: Int, desc: String): String = + result(cond, num) + " " + desc + + /** Returns a todo test result. + * + * @example `not ok 1 # TODO this doesn't work yet` + */ + def todoResult (cond: Boolean, num: Int, todo: String): String = + result(cond, num) + " # TODO " + todo + + /** Returns a todo test result that contains a description. + * + * @example `not ok 18 - test the feature # TODO can't figure this out` + */ + def todoResult (cond: Boolean, num: Int, desc: String, todo: String): String = + result(cond, num, desc) + " # TODO " + todo + + /** Returns a skipped test result with a reason. + * + * @example `ok 4 # skip this test won't run here` + */ + def skip (num: Int, reason: String): String = + "ok " + num + " # skip " + reason + + /** Returns a comment. + * + * @example `# this is a comment` + */ + def comment (message: String): String = + message.split("\n").map(m => "# " + m).mkString("\n") + + /** Returns a test plan. + * + * @example `1..5` ([[com.iinteractive.test.NumericPlan NumericPlan]]) + * @example `1..0 # SKIP don't run this test` ([[com.iinteractive.test.SkipAll SkipAll]]) + */ + def plan (plan: Plan): String = + plan.skipAll.map(m => "1..0 # SKIP " + m).getOrElse("1.." + plan.plan) + + /** Returns a bail out with a reason. + * + * @example `Bail out! Not supported on this platform.` + */ + def bailOut (message: String): String = + "Bail out! " + message +} diff --git a/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala new file mode 100644 index 0000000..1c2e88d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala @@ -0,0 +1,40 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.Plan + +/** An event emitted while parsing a TAP stream. */ +sealed trait TAPEvent + +/** The start of a TAP stream. */ +case object StartEvent extends TAPEvent + +/** The end of a TAP stream. + * @param result The [[com.iinteractive.test.tap.TAPResult TAPResult]] + * containing information about all of the tests which just + * finished running. This will be the same thing that is + * returned by the call to + * [[com.iinteractive.test.tap.Parser Parser]]'s `parse` + * method. + */ +case class EndEvent private[tap] (result: TAPResult) extends TAPEvent + +/** An individual test result. + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * containing information about the corresponding test. + */ +case class ResultEvent private[tap] (result: TestResult) extends TAPEvent + +/** A test plan. + * @param plan The [[com.iinteractive.test.Plan Plan]] corresponding to the + * line that was parsed. + */ +case class PlanEvent private[tap] (plan: Plan) extends TAPEvent + +/** The start of a subtest (currently unused). */ +case object SubtestStartEvent extends TAPEvent + +/** The end of a subtest (currently unused). */ +case class SubtestEndEvent private[tap] (result: TestResult) extends TAPEvent + +/** A comment (currently unused). */ +case class CommentEvent private[tap] (text: String) extends TAPEvent diff --git a/src/main/scala/com/iinteractive/test/tap/TAPResult.scala b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala new file mode 100644 index 0000000..c3c4926 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala @@ -0,0 +1,94 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** The summarized results of a TAP stream. Contains the + * [[com.iinteractive.test.Plan Plan]] that was given, as well as a list of + * [[com.iinteractive.test.tap.TestResult TestResults]] corresponding to each + * of the tests in the stream. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] from the TAP stream + * @param results The list of + * [[com.iinteractive.test.tap.TestResult TestResults]] from + * the TAP stream + */ +class TAPResult (val plan: Plan, val results: Seq[TestResult]) { + /** Returns true if the number of tests executed was compatible with the + * provided test plan. + */ + val matchesPlan = plan match { + case NumericPlan(n) => results.length == n + case _ => results.length == 0 + } + + /** Returns the number of test failures in the TAP stream. */ + val fails = results.count { r => + !r.passed && !r.directive.isDefined + } + + /** Returns true if all of the tests passed. */ + val testsPassed = fails == 0 + + /** Returns true if the TAP stream overall passed. + * + * Differs from `testsPassed` in that it also takes into account things + * like invalid plans. + */ + val success = plan match { + case SkipAll(_) => true + case _ => results.length > 0 && fails == 0 && matchesPlan + } + + /** Returns the exit code to use if running this test on its own. + * + * Success is indicated by 0, invalid TAP streams (such as incorrect plans) + * by 255, and other kinds of failures by the failure count. + */ + val exitCode = + if (success) { + 0 + } + else if (!matchesPlan || results.length == 0) { + 255 + } + else { + fails + } +} + +/** The result of a single test. + * + * @param passed True if the test passed + * @param number The test number in the TAP stream + * @param description The test description + * @param directive The [[com.iinteractive.test.tap.Directive Directive]] + * (either skip or todo) that was provided for this test, + * if any + * @param subtest The [[com.iinteractive.test.tap.TAPResult]] for the + * subtest that this test corresponds to, if any + */ +class TestResult ( + val passed: Boolean, + val number: Int, + val description: String, + val directive: Option[Directive], + val subtest: Option[TAPResult] +) + +/** A modifier associated with a test result. This is indicated by a `#` at + * the end of the result line, followed by the type of directive, and an + * optional message. + */ +sealed trait Directive { + val message: Option[String] +} + +/** A directive indicating that this test was skipped. */ +case class SkipDirective private[tap] ( + message: Option[String] +) extends Directive + +/** A directive indicating that this test is known to fail. */ +case class TodoDirective private[tap] ( + message: Option[String] +) extends Directive diff --git a/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala new file mode 100644 index 0000000..62fdf02 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala @@ -0,0 +1,196 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test._ + +/** This class provides a convenient yet low level API for generating TAP + * streams. Each instance of this class handles a single TAP stream, and + * keeps track of things like the current test number for you. All + * TAP-producing methods write the TAP lines to `Console.out` or + * `Console.err`, so you can override those (via `Console.withOut` or + * `Console.withErr`). + */ +class TestBuilder private ( + plan: Plan, + indent: String, + terminalInUse: Boolean +) { + plan match { + case NoPlan => () + case p => outLine(Producer.plan(p)) + } + + /** Creates a new builder instance, and emits the corresponding plan line, + * unless the plan is not given. + * + * @param plan [[com.iinteractive.test.Plan plan]] for this test. + * @param terminalInUse Whether this test is being run from a harness which + * will not just be writing directly to the output. + * This will make things written to `Console.err` have + * a newline prepended, so that they always start on + * an empty line. + */ + def this (plan: Plan = NoPlan, terminalInUse: Boolean = false) = + this(plan, "", terminalInUse) + + /** Create a new TestBuilder instance, to be used to run a subtest. This new + * instance will have all of its lines prefixed by an additional level of + * indentation. This instance will still need to have `doneTesting` + * called on it, and the result of the subtest will still need to be + * reported as a separate test result through `ok`. + */ + def cloneForSubtest (newPlan: Plan): TestBuilder = + new TestBuilder(newPlan, indent + " ", terminalInUse) + + /** Reports a single test result to `Console.out`. */ + def ok (test: Boolean) { + state.ok(test) + outLine(Producer.result(test, state.currentTest)) + } + + /** Reports a single test result with description to `Console.out`. */ + def ok (test: Boolean, description: String) { + state.ok(test) + outLine(Producer.result(test, state.currentTest, description)) + } + + /** Reports a single TODO test result to `Console.out`. */ + def todo (todo: String, test: Boolean) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, todo)) + } + + /** Reports a single TODO test result with description to `Console.out`. */ + def todo (todo: String, test: Boolean, description: String) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, description, todo)) + } + + /** Reports a single skipped test result to `Console.out`. */ + def skip (reason: String) { + state.ok(true) + outLine(Producer.skip(state.currentTest, reason)) + } + + /** Writes a comment line to `Console.err`. This will allow it to be + * visible in most summarizing harnesses (which consume and parse + * everything that goes to `Console.out`). + */ + def diag (message: String) { + errLine(Producer.comment(message)) + } + + /** Write a comment line to `Console.out`. This will typically only be + * visible in the raw TAP stream. + */ + def note (message: String) { + outLine(Producer.comment(message)) + } + + /** Abort the current test, with a message. */ + def bailOut (message: String) { + val bailOutMessage = Producer.bailOut(message) + outLine(bailOutMessage) + throw new BailOutException(bailOutMessage) + } + + /** Finalize the current builder instance. This writes the auto-calculated + * plan to `Console.out` if the plan type was `NoPlan` and reports a + * summary of the test results as a comment to `Console.err`. + * + * @return whether or not the test class as a whole passed. + */ + def doneTesting: Boolean = { + plan match { + case NumericPlan(_) => printErrors + case SkipAll(_) => () + case NoPlan => { + outLine(Producer.plan(state.currentTest)) + printErrors + } + } + state.isPassing + } + + /** The exit code to use, in harnesses that run a single test. Passing tests + * return 0, invalid tests (such as running a different number of tests + * than planned) return 255, and all others return the number of failed + * tests. + */ + def exitCode: Int = + if (state.isPassing) { + 0 + } + else if (!state.matchesPlan || state.currentTest == 0) { + 255 + } + else { + state.failCount + } + + private def printErrors { + if (!state.matchesPlan) { + val planCount = (plan match { + case NoPlan => state.currentTest + case p => p.plan + }) + val planned = planCount + " test" + (if (planCount > 1) "s" else "") + val ran = state.currentTest + diag("Looks like you planned " + planned + " but ran " + ran + ".") + } + + if (state.currentTest == 0) { + diag("No tests run!") + } + + if (state.failCount > 0) { + val count = state.failCount + val fails = count + " test" + (if (count > 1) "s" else "") + val total = + state.currentTest + (if (state.matchesPlan) "" else " run") + diag("Looks like you failed " + fails + " of " + total + ".") + } + } + + private val state = new TestState + + private def outLine (str: String) { + Console.out.println(withIndent(str)) + } + + private def errLine (str: String) { + if (terminalInUse) { + Console.err.print("\n") + } + Console.err.println(withIndent(str)) + } + + private def withIndent (str: String): String = + str.split("\n").map(s => indent + s).mkString("\n") + + private class TestState { + var passCount = 0 + var failCount = 0 + + def ok (cond: Boolean) { + if (cond) { + passCount += 1 + } + else { + failCount += 1 + } + } + + def currentTest: Int = + failCount + passCount + + def matchesPlan: Boolean = plan match { + case NumericPlan(p) => p.plan == currentTest + case _ => true + } + + def isPassing: Boolean = plan match { + case SkipAll(_) => true + case _ => currentTest > 0 && failCount == 0 && matchesPlan + } + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/package.scala b/src/main/scala/com/iinteractive/test/tap/package.scala new file mode 100644 index 0000000..312f996 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/package.scala @@ -0,0 +1,9 @@ +package com.iinteractive.test + +/** Classes for TAP generation and parsing. */ +package object tap { + /** Exception representing an error during parsing. It is thrown when a TAP + * line isn't recognized. + */ + case class ParseException (message: String) extends RuntimeException(message) +} diff --git a/src/main/scala/org/perl8/test/ExternalTest.scala b/src/main/scala/org/perl8/test/ExternalTest.scala deleted file mode 100644 index 7855aa8..0000000 --- a/src/main/scala/org/perl8/test/ExternalTest.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.iinteractive.test - -import scala.concurrent.Await -import scala.concurrent.duration.Duration -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import scala.concurrent.Future._ -import scala.annotation.tailrec - -/** Runs an external process which emits TAP, and parses it as a test. - * - * This test class can be used if you have existing tests that you would like - * to be able to include in a test suite using this framework. You just need - * to write a test class for each external test that looks like this: - * - * {{{ - * class MyTest1 extends ExternalTest("perl", "t/basic.t") - * }}} - * - * This will run your external process, and use its TAP stream as its output. - * This will allow it to, for instance, be a part of the test suite that runs - * via `sbt test`. As with any other test class, its stdout and stderr will - * be sent to `Console.out` and `Console.err`, where they can be overridden - * as needed. - */ -class ExternalTest (cmdLine: String*) extends Test { - protected def runTests (raw: Boolean): Int = { - val processBuilder = new ProcessBuilder(cmdLine: _*) - - // Ensure that if stdout and stderr are both pointing to the same place (a - // terminal or file or something) that they remain synchronized. This is - // only possible if the endpoint they are streaming to is the same place - // underneath, with no extra processing in between. (This is safe because - // stdout is typically line-buffered, and we only ever output a line at a - // time when writing TAP, so in theory, buffering of the underlying file - // descriptor shouldn't make a difference here) - if (Console.out eq System.out) { - processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT) - } - if (Console.err eq System.err) { - processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT) - } - - val process = processBuilder.start - - val streams = Seq( - Console.out -> process.getInputStream, - Console.err -> process.getErrorStream - ) - - val listeners = streams.map { case (out, in) => - Future { - val buf = new Array[Byte](1024) - - @tailrec - def read { - val bytes = in.read(buf) - if (bytes >= 0) { - out.print(new String(buf.take(bytes))) - read - } - } - - read - true - } - } - - val exitCode = process.waitFor - Await.ready(Future.sequence(listeners), Duration.Inf) - exitCode - } -} diff --git a/src/main/scala/org/perl8/test/Test.scala b/src/main/scala/org/perl8/test/Test.scala deleted file mode 100644 index 2161de3..0000000 --- a/src/main/scala/org/perl8/test/Test.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.iinteractive.test - -/** Base trait for test classes in this framework. Any tests that should be - * autodiscovered by `sbt test` should extend this trait, and implement - * [[runTests]]. - */ -trait Test { - /** Runs the test. The TAP stream will be written to Console.out and - * Console.err, so you can swap these out as required in order to parse it. - * - * @return The exit code that the test produced. Success is indicated by 0, - * failure to run the correct number of tests by 255, and any other - * failure by the number of tests that failed. This should be used - * by reporters which run a single test, which can call - * `sys.exit(exitCode)` - */ - def run: Int = - runTests(false) - - /** Runs the test just like [[run]], but in a way that makes sense when test - * results are being summarized rather than directly displayed. - * - * Summarizing test reporters tend to repeatedly update the same line on - * the terminal, so this method makes calls to - * [[com.iinteractive.test.tap.TestBuilder#diag diag]] (which sends - * messages to stderr, where they are typically displayed as-is) prefix the - * message with a newline, to ensure that the output starts on its own - * line. - */ - def runInHarness: Int = - runTests(true) - - protected def runTests (terminalInUse: Boolean): Int -} diff --git a/src/main/scala/org/perl8/test/TestMore.scala b/src/main/scala/org/perl8/test/TestMore.scala deleted file mode 100644 index eb5210b..0000000 --- a/src/main/scala/org/perl8/test/TestMore.scala +++ /dev/null @@ -1,519 +0,0 @@ -package com.iinteractive.test - -import scala.util.matching.Regex - -import com.iinteractive.test.tap.TestBuilder - -/** This class is an implementation of the excellent - * [[https://metacpan.org/module/Test::More Test::More]] testing library for - * Perl. It provides a simple assertion-based testing API, which produces - * [[http://en.wikipedia.org/wiki/Test_Anything_Protocol TAP]], which can be - * parsed by any TAP consumer. This library includes several TAP-consuming - * harnesses to use with tests using this class, including one that supports - * testing via `sbt test`. - * - * ==Basics== - * - * The most basic test looks like this: - * - * 
-  * class MyTest extends TestMore {
-  *   ok(true)
-  * }
-  *
- * - * This runs a test containing a single assertion. This will generate a TAP - * stream that looks like this: - * - * 
-  * ok 1
-  * 1..1
-  *
- * - * which can be parsed by one of the test harnesses provided by this library. - * - * ==Running tests== - * - * The simplest way to run tests is through sbt. You can register this - * framework with sbt by adding this line to your `build.sbt` file: - * - * 
-  * testFrameworks += new TestFramework("com.iinteractive.test.sbt.Framework")
-  *
- * - * Then, any classes in your test directory which extend `TestMore` will be - * automatically detected and run. - * - * ==Assertions== - * - * This class contains many more assertion methods than just `ok`. Here is a - * more extensive example (borrowed from Test::More's documentation): - * - * 
-  * class MyTest extends TestMore {
-  *   ok(got == expected, testName)
-  *
-  *   is(got, expected, testName)
-  *   isnt(got, expected, testName)
-  *
-  *   diag("here's what went wrong")
-  *
-  *   like(got, """expected""".r, testName)
-  *   unlike(got, """expected""".r, testName)
-  *
-  *   skip(howMany, why) {
-  *     ok(foo(), testName)
-  *     is(foo(42), 23, testName)
-  *   }
-  *
-  *   todo(why) {
-  *     ok(foo(), testName)
-  *     is(foo(42), 23, testName)
-  *   }
-  *
-  *   pass(testName)
-  *   fail(testName)
-  *
-  *   BAIL_OUT(why)
-  * }
-  *
- * - * The difference between the simple `ok` method and the more specific - * methods like `is` and `like` is in how failures are reported. If you write - * this: - * - * 
-  * ok(1 == 2)
-  *
- * - * the output will look like this: - * - * 
-  * not ok 1
-  * #   Failed test at MyTest.scala line 4.
-  *
- * - * On the other hand, a more specific assertion such as: - * - * 
-  * is(1, 2)
-  *
- * - * will produce more useful output: - * - * 
-  * not ok 1
-  * #   Failed test at MyTest.scala line 4.
-  * #          got: 1
-  * #     expected: 2
-  *
- * - * In addition to assertions, there are also several methods which take a - * block of code to run, to modify the assertions contained in that block. - * - * The `todo` method runs tests which are expected to fail. If they do fail, - * the failure is reported to the test harness as a normal succeeding test, - * and nothing happens. If they succeed, they are still reported as a - * succeeding test, but a message is displayed to the user indicating that - * the todo indication can be removed. - * - * The `skip` method takes a block which should not be run at all. This is - * similar to `todo`, except that it is useful for tests which could cause - * problems if they were to actually run. Since the tests are never run, it's - * not possible to count how many tests there should be, so this must be - * specified as a parameter. - * - * The `subtest` method runs a block of assertions as though they were an - * entirely separate test, and then reports the result of that test as a - * single assertion in the test that called `subtest`. - * - * ==Test plans== - * - * Normally, you can run any number of assertions within your class body, and - * the framework will assume that if no exceptions were thrown, all of the - * assertions that were meant to be run were actually run. Sometimes, - * however, that may not be a safe assumption, especially with heavily - * callback-driven code. In this case, you can specify exactly how many tests - * you intend to run, and the number of tests actually run will be checked - * against this at the end. To declare this, give a number to the `TestMore` - * constructor: - * - * 
-  * class MyTest extends TestMore(5) {
-  *   ???
-  * }
-  *
- * - * In addition, if the entire test should be skipped, you can give a plan of - * `SkipAll()`: - * - * 
-  * class MyTest extends TestMore(SkipAll()) {
-  *   ???
-  * }
-  *
- * - * ==Extensions== - * - * These assertion methods are written with the intention of being - * composable. You can write your own test methods which call `is` or `ok` on - * more specific bits of data. The one issue here is that, as shown above, - * test failure messages refer to the file and line where the `is` or `ok` - * call was made. If you want this to instead point at the line where your - * assertion helper method was called, you can use the `hideTestMethod` - * method like this: - * - * 
-  * trait MyTestHelpers { this: TestMore =>
-  *   def notok (cond: Boolean) = hideTestMethod {
-  *     ok(!cond)
-  *   }
-  * }
-  *
- * - * This way, the test failure will be reported from the line where `notok` - * was called, not from the call to `ok` in the `notok` method. - */ -class TestMore (plan: Plan = NoPlan) extends Test with DelayedInit { - def delayedInit (body: => Unit) { - testBody = { terminalInUse => - todo = None - builder = new TestBuilder(plan, terminalInUse) - plan match { - case SkipAll(_) => () - case _ => body - } - } - } - - protected def runTests (terminalInUse: Boolean): Int = { - if (testBody == null) { - delayedInit { } - } - - testBody(terminalInUse) - builder.doneTesting - builder.exitCode - } - - /** Assert that a condition is true. - * - * @example `ok(response.isSuccess)` - */ - def ok (cond: Boolean): Boolean = - test(cond) - - /** Assert that a condition is true, and describe the assertion. - * - * @example `ok(response.isSuccess, "the response succeeded")` - */ - def ok (cond: Boolean, desc: String): Boolean = - testWithDesc(cond, desc) - - /** Assert that two objects are equal (using `==`). - * - * @example `is(response.status, 200)` - */ - def is[T] (got: T, expected: T): Boolean = - test(got == expected, isMessage(got, expected)) - - /** Assert that two objects are equal (using `==`), and describe the - * assertion. - * - * @example `is(response.status, 200, "we got a 200 OK response")` - */ - def is[T] (got: T, expected: T, desc: String): Boolean = - testWithDesc(got == expected, desc, isMessage(got, expected)) - - /** Assert that two objects are not equal (using `!=`). - * - * @example `isnt(response.body, "")` - */ - def isnt[T] (got: T, expected: T): Boolean = - test(got != expected, isntMessage(got)) - - /** Assert that two objects are not equal (using `!=`), and describe the - * assertion. - * - * @example `isnt(response.body, "", "we got a response body")` - */ - def isnt[T] (got: T, expected: T, desc: String): Boolean = - testWithDesc(got != expected, desc, isntMessage(got)) - - /** Assert that a string matches a regular expression. - * - * @example `like(response.header("Content-Type"), """text/x?html""".r)` - */ - def like (got: String, rx: Regex): Boolean = - test(rx.findFirstIn(got).nonEmpty, likeMessage(got, rx)) - - /** Assert that a string matches a regular expression, and describe the - * assertion. - * - * @example `like(response.header("Content-Type"), """text/x?html""".r, "we got an html content type")` - */ - def like (got: String, rx: Regex, desc: String): Boolean = - testWithDesc(rx.findFirstIn(got).nonEmpty, desc, likeMessage(got, rx)) - - /** Assert that a string doesn't match a regular expression. - * - * @example `unlike(response.header("Authorization"), """^Digest.*""".r)` - */ - def unlike (got: String, rx: Regex): Boolean = - test(rx.findFirstIn(got).isEmpty, unlikeMessage(got, rx)) - - /** Assert that a string doesn't match a regular expression. - * - * @example `unlike(response.header("Authorization"), """^Digest.*""".r, "we don't support digest authentication")` - */ - def unlike (got: String, rx: Regex, desc: String): Boolean = - testWithDesc(rx.findFirstIn(got).isEmpty, desc, unlikeMessage(got, rx)) - - /** An assertion that always succeeds. - * - * @example `pass()` - */ - def pass: Boolean = - ok(true) - - /** An assertion that always succeeds, with a reason. - * - * @example `pass("this line of code should be executed")` - */ - def pass (desc: String): Boolean = - ok(true, desc) - - /** An assertion that always fails. - * - * @example `fail()` - */ - def fail: Boolean = - ok(false) - - /** An assertion that always fails, with a reason. - * - * @example `fail("we should never get here")` - */ - def fail (desc: String): Boolean = - ok(false, desc) - - /** Output a comment to `Console.err`. This is intended to be visible to - * users even when running the test under a summarizing harness. - * - * @example `diag("Testing with Scala " + util.Properties.versionString)` - */ - def diag (message: String) { - builder.diag(message) - } - - /** Output a comment to `Console.out`. This is intended to only be visible - * when viewing the raw TAP stream. - * - * @example `note("Starting the response tests")` - */ - def note (message: String) { - builder.note(message) - } - - /** Halt execution of the entire test suite. - * - * @example `BAIL_OUT("can't connect to the database!")` - */ - def BAIL_OUT (desc: String) { - builder.bailOut(desc) - } - - /** Mark a block of tests as expected to fail. If the tests which run in the - * todo block fail, they will not be treated as test failures, and if they - * succeed, the user will be notified. - * - * @example `todo("waiting on fixes elsewhere") { ??? }` - */ - def todo (reason: String)(body: => Unit) { - val oldTodo = todo - try { - todo = Some(reason) - body - } - finally { - todo = oldTodo - } - } - - /** Mark a block of tests that should not be run at all. They are treated as - * always passing. - * - * @example `skip(3, "too dangerous to run for now") { ??? }` - */ - def skip (count: Int, reason: String)(body: => Unit) { - for (i <- 1 to count) { - builder.skip(reason) - } - } - - /** Declare a logical group of assertions, to be run as a single test. This - * is effectively an entirely separate test, which is run, and the result - * of that test is reported as a single assertion in the test that contains - * it. The subtest can specify its own plan in the same way that the - * overall test is allowed to. The name will be used as the description for - * the single assertion that the overall test sees. - * - * @example `subtest("response tests") { ??? }` - */ - def subtest ( - name: String, - plan: Plan = NoPlan - )(body: => Unit): Boolean = { - val oldBuilder = builder - val success = try { - builder = oldBuilder.cloneForSubtest(plan) - body - builder.doneTesting - } - finally { - builder = oldBuilder - } - ok(success, name) - } - - /** A helper method which should be used to wrap test utility methods. - * Normally, when tests fail, a message is printed giving the file and line - * number of the call to the test method. If you write your own test - * methods, they will typically use the existing methods to generate - * assertions, and so the file and line numbers will likely be much less - * useful. Wrapping the body of your method in this method will ensure that - * the file and line number that is reported is the line where your helper - * method is called instead. - * - * @example `def testFixtures = hideTestMethod { ??? }` - */ - def hideTestMethod[T] (body: => T): T = { - // this just adds a method call with a known name to the stack trace, so - // that we can detect it later - body - } - - private def isMessage[T] (got: T, expected: T): String = - " got: '" + got + "'\n" + - " expected: '" + expected + "'\n" - - private def isntMessage[T] (got: T): String = - " got: '" + got + "'\n" + - " expected: anything else\n" - - private def likeMessage (got: String, rx: Regex): String = - " '" + got + "'\n" + - " doesn't match '" + rx + "'\n" - - private def unlikeMessage (got: String, rx: Regex): String = - " '" + got + "'\n" + - " matches '" + rx + "'\n" - - private def testWithDesc ( - cond: Boolean, - desc: String - ): Boolean = { - todo match { - case Some(t) => builder.todo(t, cond, "- " + desc) - case None => builder.ok(cond, "- " + desc) - } - if (!cond) { - failed(Some(desc), None) - } - cond - } - - private def testWithDesc ( - cond: Boolean, - desc: String, - reason: => String - ): Boolean = { - todo match { - case Some(t) => builder.todo(t, cond, "- " + desc) - case None => builder.ok(cond, "- " + desc) - } - if (!cond) { - failed(Some(desc), Some(reason)) - } - cond - } - - private def test (cond: Boolean): Boolean = { - todo match { - case Some(t) => builder.todo(t, cond) - case None => builder.ok(cond) - } - if (!cond) { - failed(None, None) - } - cond - } - - private def test (cond: Boolean, reason: => String): Boolean = { - todo match { - case Some(t) => builder.todo(t, cond) - case None => builder.ok(cond) - } - if (!cond) { - failed(None, Some(reason)) - } - cond - } - - private def failed (desc: Option[String], reason: Option[String]) { - val stack = Thread.currentThread.getStackTrace.drop(1).filter { frame => - !ignoreFrame(frame) - } - val idx = stack.lastIndexWhere { frame => - frame.getClassName == "com.iinteractive.test.TestMore" && - frame.getMethodName == "hideTestMethod" - } - val caller = idx match { - case -1 => stack.headOption - // one level to jump out of hideTestMethod and one level to jump out of - // the method that called hideTestMethod - case i => stack.drop(i + 2).headOption - - } - val (file, line) = caller match { - case Some(frame) => (frame.getFileName, frame.getLineNumber) - case None => ("", "") - } - val message = " " + (todo match { - case Some(_) => "Failed (TODO) test" - case None => "Failed test" - }) + (desc match { - case Some(m) => " '" + m + "'\n " - case None => " " - }) - val trace = "at " + file + " line " + line + "." - val explanation = message + trace + reason.map("\n" + _).getOrElse("") - if (todo.isDefined) { - builder.note(explanation) - } - else { - builder.diag(explanation) - } - } - - protected def ignoreFrame (frame: StackTraceElement): Boolean = { - val className = frame.getClassName - val methodName = frame.getMethodName - - // ignore everything in this class, except the hideTestMethod call which we - // use as a stack trace marker - (className == "com.iinteractive.test.TestMore" && - methodName != "hideTestMethod") || - // when you call a method in a class when the method is defined in a - // trait, it calls a stub which calls the real definition in the trait. - // the trait is represented under the hood as a class with the same name - // as the trait, except with $class appended. this is a gross reliance on - // implementation details that could change at any moment, but i don't - // really see any better options. - """\$class$""".r.findFirstIn(className).nonEmpty - } - - private var todo: Option[String] = _ - private var builder: TestBuilder = _ - private var testBody: Boolean => Unit = _ -} diff --git a/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala b/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala deleted file mode 100644 index 56f32fd..0000000 --- a/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.iinteractive.test.harness - -/** Classes that implement `MultiTestReporter` are capable of running a group - * of test classes, given their names. This typically involves some sort of - * summarization. - * - * @see [[com.iinteractive.test.harness.Reporter Reporter]]. - */ -trait MultiTestReporter { - /** Runs the test classes identifed by the list of fully qualified class - * names `testNames`. - * - * @return The exit code for the harness to use. Will be 0 on success. - */ - def run (testNames: Seq[String]): Int -} diff --git a/src/main/scala/org/perl8/test/harness/Reporter.scala b/src/main/scala/org/perl8/test/harness/Reporter.scala deleted file mode 100644 index a47444b..0000000 --- a/src/main/scala/org/perl8/test/harness/Reporter.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.iinteractive.test.harness - -/** Classes that implement `Reporter` are capable of running a test class, - * given its name. - * - * @see [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]]. - */ -trait Reporter { - /** Runs the test class identifed by the fully qualified class name - * `testName`. - * - * @return The exit code for the harness to use. Will be 0 on success. - */ - def run (testName: String): Int -} diff --git a/src/main/scala/org/perl8/test/harness/SummarizedTests.scala b/src/main/scala/org/perl8/test/harness/SummarizedTests.scala deleted file mode 100644 index cd555dd..0000000 --- a/src/main/scala/org/perl8/test/harness/SummarizedTests.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.iinteractive.test.harness - -import java.io.{PipedInputStream,PipedOutputStream} -import scala.concurrent.Await -import scala.concurrent.duration.Duration -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future - -import com.iinteractive.test.tap.{Parser,TAPEvent,TAPResult,TodoDirective} -import com.iinteractive.test.Test - -/** This is a trait for classes that run tests and summarize the results. It - * provides a single `runOneTest` method, which runs a test class and - * produces a stream of [[com.iinteractive.test.tap.TAPEvent TAP events]] - * which can be used to produce whatever summarized output you need. - */ -trait SummarizedTests { - /** Runs a single [[com.iinteractive.test.Test test]] instance, calling `cb` - * with each [[com.iinteractive.test.tap.TAPEvent TAP event]] as it is - * produced. - * - * @return The overall result of the test instance. - */ - protected def runOneTest (test: Test, cb: TAPEvent => Unit): TAPResult = { - val out = new PipedOutputStream - val in = new PipedInputStream(out) - - val testFuture = Future { - Console.withOut(out) { - test.runInHarness - } - out.close - } - - val parser = new Parser(cb) - val result = parser.parse(in) - in.close - Await.ready(testFuture, Duration.Inf) - - result - } -} diff --git a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala deleted file mode 100644 index a5fe1e0..0000000 --- a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala +++ /dev/null @@ -1,192 +0,0 @@ -package com.iinteractive.test.harness - -import com.iinteractive.test.tap.{TAPEvent,TAPResult,TodoDirective} -import com.iinteractive.test.tap.{StartEvent,ResultEvent,PlanEvent,EndEvent} -import com.iinteractive.test.Test - -/** Runs a series of tests. The TAP output from these tests is parsed, and - * output is produced which is similar in style to Perl's - * [[https://metacpan.org/module/Test::Harness Test::Harness]]. - */ -class SummaryReporter extends MultiTestReporter with SummarizedTests { - def run (testNames: Seq[String]): Int = { - val results = runTests(testNames) - val success = results.values.forall(_.success) - printTestSummary(success, results) - if (success) 0 else 1 - } - - protected def runTests (testNames: Seq[String]): Map[String, TAPResult] = { - val maxLength = testNames.map(_.length).max - - testNames.map { name => - val callbackGenerator: () => TAPEvent => Unit = () => { - var width = 0 - var tests = 0 - var plan: Option[Int] = None - - def status = { - tests + "/" + plan.getOrElse("?") - } - - def printStatus (st: String) { - print("\r" + (" " * width) + "\r") - val line = - name + " " + ("." * (maxLength - name.length)) + ".. " + st - width = line.length - print(line) - Console.out.flush - } - - (e: TAPEvent) => e match { - case StartEvent => { - printStatus("") - } - case PlanEvent(p) => { - plan = Some(p.plan) - printStatus(status) - } - case ResultEvent(r) => { - tests += 1 - printStatus(status) - } - case EndEvent(result) => { - if (result.success) { - printStatus("") - println("ok") - } - else { - val results = result.results.length - val failed = result.results.count { t => - !t.passed && !t.directive.isDefined - } - - printStatus("") - println("Dubious, test returned " + result.exitCode) - println("Failed " + failed + "/" + results + " subtests") - } - } - case _ => () - } - } - - name -> runOneTest(newInstance[Test](name), callbackGenerator()) - }.toMap - } - - protected def printTestSummary ( - success: Boolean, - results: Map[String, TAPResult] - ) { - printSuccess(success) - printLongSummary(results) - printShortSummary(results) - printPassFail(success, results) - } - - private def printSuccess (success: Boolean) { - if (success) { - println("All tests successful.") - } - } - - private def printShortSummary (results: Map[String, TAPResult]) { - val files = results.size - val tests = results.values.map(_.results.length).sum - println("Files=" + files + ", Tests=" + tests) - } - - private def printLongSummary (results: Map[String, TAPResult]) { - val todoSucceeded = results.mapValues { r => - r.results.filter { t => - t.directive match { - case Some(TodoDirective(_)) => t.passed - case _ => false - } - } - }.filter(_._2.length > 0) - - val testsFailed = results.mapValues { r => - r.results.filter { t => - t.directive match { - case None => !t.passed - case _ => false - } - } - }.filter(_._2.length > 0) - - val testNames = (todoSucceeded ++ testsFailed).keys - - if (testNames.nonEmpty) { - println("") - println("Test Summary Report") - println("-------------------") - - val maxLength = testNames.map(_.length).max - - for (name <- testNames) { - val result = results(name) - - println( - name + (" " * (maxLength - name.length)) + " " + - "(Tests: " + result.results.length + " " + - "Failed: " + testsFailed.getOrElse(name, Nil).length + ")" - ) - - if (testsFailed.isDefinedAt(name)) { - val fails = testsFailed(name) - println( - " Failed test" + (if (fails.length > 1) "s" else "") + ": " + - fails.map(_.number).mkString(", ") - ) - } - - if (todoSucceeded.isDefinedAt(name)) { - val todos = todoSucceeded(name) - println( - " TODO passed: " + - todos.map(_.number).mkString(", ") - ) - } - - val exitCode = results(name).exitCode - if (exitCode != 0) { - println(" Non-zero exit status: " + exitCode) - } - } - } - } - - private def printPassFail ( - success: Boolean, - results: Map[String, TAPResult] - ) { - if (success) { - println("Result: PASS") - } - else { - println("Result: FAIL") - - val testResults = results.values - - val testsFailed = testResults.map { r => - r.results.count { t => - t.directive match { - case None => !t.passed - case _ => false - } - } - }.filter(_ > 0) - val failedFiles = testsFailed.size - val failedTests = testsFailed.sum - - val allFiles = testResults.size - val allTests = testResults.map(_.results.length).sum - - println( - "Failed " + failedFiles + "/" + allFiles + " test programs. " + - failedTests + "/" + allTests + " subtests failed." - ) - } - } -} diff --git a/src/main/scala/org/perl8/test/harness/TAPReporter.scala b/src/main/scala/org/perl8/test/harness/TAPReporter.scala deleted file mode 100644 index 8a4dc9f..0000000 --- a/src/main/scala/org/perl8/test/harness/TAPReporter.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.iinteractive.test.harness - -import com.iinteractive.test.tap -import com.iinteractive.test.Test - -/** Runs a single test. The TAP stream from that test is written directly to - * stdout/stderr. - */ -class TAPReporter extends Reporter { - def run (testName: String): Int = - newInstance[Test](testName).run -} diff --git a/src/main/scala/org/perl8/test/harness/TestHarness.scala b/src/main/scala/org/perl8/test/harness/TestHarness.scala deleted file mode 100644 index 7b8f4ae..0000000 --- a/src/main/scala/org/perl8/test/harness/TestHarness.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.iinteractive.test.harness - -/** This is the entry point to running tests written with this library from - * the command line. Note that this library also implements the - * [[https://github.com/harrah/test-interface common testing interface]] for - * test libraries, so tests should also just work with `sbt test`. - * - * If this application is run and given just a single test class name, it - * will run that test and write its TAP stream to the console. - * - * {{{ - * $ scala com.iinteractive.test.harness.TestHarness MyTest - * ok 1 - * ok 2 - * 1..2 - * }}} - * - * If this application is run and given multiple test class names, it will - * run each of those tests, and present a summary report, similar to the one - * produces by - * [[https://metacpan.org/module/Test::Harness Perl's Test::Harness]]. - * - * {{{ - * $ scala com.iinteractive.test.harness.TestHarness MyTest1 MyTest2 - * MyTest1 .. ok - * MyTest2 .. ok - * All tests successful. - * Files=2, Tests=4 - * Result: PASS - * }}} - * - * This application also accepts a few command line options to customize its - * behavior: - * - * - `-r`: Alternative [[com.iinteractive.test.harness.Reporter Reporter]] - * class to use for running a single test. - * - `-R`: Alternative - * [[com.iinteractive.test.harness.MultiTestReporter MultiTestReporter]] - * class to use for running a group of tests. Also enables using the - * MultiTestReporter for a single test. - * - `--help`: Prints usage information. - */ -object TestHarness { - import com.iinteractive.test.Test - - /** Entry point for the harness application. */ - def main (args: Array[String]) { - val opts = parseOpts(args.toList) - val single = opts("prefer-single").asInstanceOf[Boolean] - - val exitCode = if (single) { - val reporterName = opts("single-reporter").asInstanceOf[String] - val testName = opts("test-classes").asInstanceOf[List[String]].apply(0) - val reporter = newInstance[Reporter](reporterName) - reporter.run(testName) - } - else { - val reporterName = opts("multi-reporter").asInstanceOf[String] - val testNames = opts("test-classes").asInstanceOf[List[String]] - val reporter = newInstance[MultiTestReporter](reporterName) - reporter.run(testNames) - } - - sys.exit(exitCode) - } - - protected def parseOpts (args: List[String]): Map[String, Any] = args match { - case Nil => Map( - "single-reporter" -> "com.iinteractive.test.harness.TAPReporter", - "multi-reporter" -> "com.iinteractive.test.harness.SummaryReporter", - "prefer-single" -> true, - "test-classes" -> Nil - ) - - case "-r" :: singleReporter :: rest => - parseOpts(rest) + ("single-reporter" -> singleReporter) - - case "-R" :: multiReporter :: rest => - parseOpts(rest) ++ Map( - "multi-reporter" -> multiReporter, - "prefer-single" -> false - ) - - case "--help" :: rest => - usage(0) - - case `unknownOption` :: rest => - usage(1) - - case testClass :: rest => { - val opts = parseOpts(rest) - val tests = opts("test-classes").asInstanceOf[List[String]] - opts ++ Map( - "test-classes" -> (testClass :: tests), - "prefer-single" -> tests.isEmpty - ) - } - } - - protected def usage (exitCode: Int) = { - val out = if (exitCode == 0) Console.out else Console.err - out.println("harness [-r ]\n" + - " [-R ]\n" + - " [--help]\n" + - " [...]\n") - sys.exit(exitCode) - } - - private val unknownOption = """^-.*""".r -} diff --git a/src/main/scala/org/perl8/test/harness/package.scala b/src/main/scala/org/perl8/test/harness/package.scala deleted file mode 100644 index 1b74f9d..0000000 --- a/src/main/scala/org/perl8/test/harness/package.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.iinteractive.test - -/** Classes to handle running test instances and providing output. */ -package object harness { - import scala.reflect.{ClassTag,classTag} - - /** Loads `className`, returning the - * [[http://docs.oracle.com/javase/7/docs/api/java/lang/Class.html Class]] - * instance. - */ - def loadClass[T: ClassTag] (className: String): Class[_] = - classTag[T].runtimeClass.getClassLoader.loadClass(className) - - /** Loads `className` and creates a new instance of it, using the - * no-argument constructor. - */ - def newInstance[T: ClassTag] (className: String): T = - loadClass[T](className).newInstance.asInstanceOf[T] - - /** Loads `className` and creates a new instance of it, using a - * one-argument constructor. Passes `arg` as the argument to the - * constructor. - */ - def newInstance[T: ClassTag, U <: AnyRef: ClassTag] ( - className: String, - arg: U - ): T = { - val classObj = loadClass[T](className) - val argClassObj = classTag[U].runtimeClass - val constructor = classObj.getConstructor(argClassObj) - constructor.newInstance(arg).asInstanceOf[T] - } -} diff --git a/src/main/scala/org/perl8/test/package.scala b/src/main/scala/org/perl8/test/package.scala deleted file mode 100644 index df077f2..0000000 --- a/src/main/scala/org/perl8/test/package.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.iinteractive - -/** This library implements several components of Perl's testing ecosystem in - * Scala. The most useful place to get started to use this library is likely - * [[com.iinteractive.test.TestMore TestMore]]. - */ -package object test { - import language.implicitConversions - - /** Converts an - * [[http://www.scala-lang.org/api/current/index.html#scala.Int Int]] to a - * [[NumericPlan]]. - */ - implicit def intToPlan (p: Int): Plan = - NumericPlan(p) - - /** A test plan. This represents the TAP statement telling how many tests - * will be run. - */ - sealed trait Plan { - /** How many tests will be run. */ - val plan: Int - /** Whether this test was skipped. It should contain `Some(message)` if - * the test is skipped, and `None` otherwise. - */ - val skipAll: Option[String] - } - - /** An explicit plan number. Corresponds to `1..5` in TAP. */ - case class NumericPlan (plan: Int) extends Plan { - /** @inheritdoc - * - * Always `None` for this class. - */ - val skipAll = None - } - - /** A test which did not declare a plan yet. */ - case object NoPlan extends Plan { - /** @inheritdoc - * - * Always 0 for this class. - */ - val plan = 0 - /** @inheritdoc - * - * Always `None` for this class. - */ - val skipAll = None - } - - /** A test which has declared that the entire test has been skipped. - * Corresponds to `1..0 # SKIP [message]` in TAP. - */ - case class SkipAll (message: String) extends Plan { - /** @inheritdoc - * - * Always 0 for this class. - */ - val plan = 0 - /** @inheritdoc - * - * Never `None` for this class. - */ - val skipAll = Some(message) - } - - /** Exception thrown when a test bails out. Corresponds to - * `Bail out! [message]` in TAP. - */ - case class BailOutException ( - message: String - ) extends RuntimeException(message) -} diff --git a/src/main/scala/org/perl8/test/sbt/Fingerprint.scala b/src/main/scala/org/perl8/test/sbt/Fingerprint.scala deleted file mode 100644 index bab13c5..0000000 --- a/src/main/scala/org/perl8/test/sbt/Fingerprint.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.iinteractive.test.sbt - -import org.scalatools.testing - -/** Implementation of - * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Fingerprint.java org.scalatools.testing.Fingerprint]]. - */ -object Fingerprint extends testing.SubclassFingerprint { - def isModule: Boolean = false - def superClassName: String = "com.iinteractive.test.Test" -} diff --git a/src/main/scala/org/perl8/test/sbt/Framework.scala b/src/main/scala/org/perl8/test/sbt/Framework.scala deleted file mode 100644 index bb3d0bb..0000000 --- a/src/main/scala/org/perl8/test/sbt/Framework.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.iinteractive.test.sbt - -import org.scalatools.testing - -/** Implementation of - * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Framework.java org.scalatools.testing.Framework]]. - */ -class Framework extends testing.Framework { - val name: String = "Perl8 Test" - val tests: Array[testing.Fingerprint] = Array(Fingerprint) - - def testRunner ( - testClassLoader: ClassLoader, - loggers: Array[testing.Logger] - ): testing.Runner = { - new Runner(testClassLoader, loggers) - } -} diff --git a/src/main/scala/org/perl8/test/sbt/Runner.scala b/src/main/scala/org/perl8/test/sbt/Runner.scala deleted file mode 100644 index 0eee4cf..0000000 --- a/src/main/scala/org/perl8/test/sbt/Runner.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.iinteractive.test.sbt - -import org.scalatools.testing - -import com.iinteractive.test.harness.SummaryReporter -import com.iinteractive.test.Test - -/** Implementation of - * [[http://github.com/harrah/test-interface/blob/master/src/org/scalatools/testing/Runner2.java org.scalatools.testing.Runner2]] - * using [[com.iinteractive.test.sbt.SBTReporter SBTReporter]]. - */ -class Runner ( - loader: ClassLoader, - loggers: Array[testing.Logger] -) extends testing.Runner2 { - def run ( - testClassName: String, - fingerprint: testing.Fingerprint, - eventHandler: testing.EventHandler, - args: Array[String] - ) { - val reporter = new SBTReporter(loader, loggers, eventHandler) - reporter.run(testClassName) - } -} diff --git a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala b/src/main/scala/org/perl8/test/sbt/SBTReporter.scala deleted file mode 100644 index 34df60d..0000000 --- a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.iinteractive.test.sbt - -import org.scalatools.testing - -import com.iinteractive.test.harness.{Reporter,SummarizedTests} -import com.iinteractive.test.tap.{TAPEvent,ResultEvent,EndEvent} -import com.iinteractive.test.Test - -/** Runs a single test under the SBT test harness. */ -class SBTReporter ( - loader: ClassLoader, - loggers: Array[testing.Logger], - eventHandler: testing.EventHandler -) extends Reporter with SummarizedTests { - def run (testName: String): Int = { - val cb = (e: TAPEvent) => e match { - case ResultEvent(r) => { - val event = new testing.Event { - val testName: String = r.description - val description: String = r.description - val result: testing.Result = - if (r.passed) { - testing.Result.Success - } - else if (r.directive.isDefined) { - testing.Result.Skipped - } - else { - testing.Result.Failure - } - val error: Throwable = null - } - eventHandler.handle(event) - } - case EndEvent(result) => { - val testsPassed = result.success - val correctCode = result.exitCode == 0 - val event = new testing.Event { - val testName: String = "exit code is 0" - val description: String = "exit code is 0" - val result: testing.Result = - if (correctCode) { - testing.Result.Success - } - else { - testing.Result.Failure - } - val error: Throwable = null - } - eventHandler.handle(event) - - if (testsPassed && correctCode) { - logInfo("PASS " + testName) - } - else { - val results = result.results.length - val failed = result.results.count { t => - !t.passed && !t.directive.isDefined - } - - val errors = Seq( - (if (testsPassed) - None - else - Some("failed " + failed + "/" + results)), - (if (correctCode) - None - else - Some("non-zero exit code: " + result.exitCode)) - ).flatten.mkString("(", ", ", ")") - - logError("FAIL " + testName + " " + errors) - } - } - case _ => () - } - - runOneTest( - loader.loadClass(testName).newInstance.asInstanceOf[Test], - cb - ).exitCode - } - - private def logDebug (msg: String) { - loggers.foreach(_.debug(msg)) - } - - private def logInfo (msg: String) { - loggers.foreach(_.info(msg)) - } - - private def logWarn (msg: String) { - loggers.foreach(_.warn(msg)) - } - - private def logError (msg: String) { - loggers.foreach(_.error(msg)) - } -} diff --git a/src/main/scala/org/perl8/test/sbt/package.scala b/src/main/scala/org/perl8/test/sbt/package.scala deleted file mode 100644 index eeb1f68..0000000 --- a/src/main/scala/org/perl8/test/sbt/package.scala +++ /dev/null @@ -1,4 +0,0 @@ -package com.iinteractive.test - -/** Classes for interoperating with `sbt test`. */ -package object sbt diff --git a/src/main/scala/org/perl8/test/tap/Consumer.scala b/src/main/scala/org/perl8/test/tap/Consumer.scala deleted file mode 100644 index 09a2f0a..0000000 --- a/src/main/scala/org/perl8/test/tap/Consumer.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.iinteractive.test.tap - -import com.iinteractive.test.{Plan,NumericPlan,SkipAll} - -/** Contains a method to parse an individual line of TAP. */ -object Consumer { - /** Parses a line of TAP. - * - * @return A [[com.iinteractive.test.tap.Consumer.Line Line]] object - * corresponding to the parsed line. - */ - def parseLine (line: String): Line = { - commentRx.findFirstMatchIn(line).map { m => - m.subgroups match { - case Seq(indent, text) => new CommentLine(text, indent) - } - }.getOrElse { - planRx.findFirstMatchIn(line).map { m => - m.subgroups match { - case Seq(indent, p, null) => - new PlanLine(NumericPlan(p.toInt), indent) - case Seq(indent, _, skip) => - new PlanLine(SkipAll(skip), indent) - } - }.getOrElse { - resultRx.findFirstMatchIn(line).map { m => - val indent = m.group(1) - val passed = m.group(2) == null - val number = m.group(3).toInt - val description = m.group(4) match { - case null => "" - case s => s.trim - } - val directive = (m.group(5), m.group(6)) match { - case (null, null) => None - case (d, r) => { - val reason = if (r == null) "" else r - """(?i:skip)""".r.findFirstIn(d) match { - case Some(_) => Some(new SkipDirective(Some(reason))) - case None => Some(new TodoDirective(Some(reason))) - } - } - } - val result = new TestResult( - passed, - number, - description, - directive, - None - ) - new ResultLine(result, indent) - }.getOrElse { - throw ParseException("Couldn't parse line: " + line) - } - } - } - } - - /** The representation of a parsed line of TAP. */ - sealed trait Line { - /** The meaningful portion of the TAP line. */ - def contents: String - /** The indentation of the TAP line (used in subtests). */ - def indent: String - - /** The line itself that was parsed. */ - override def toString: String = - indent + contents - } - - /** A parsed TAP line containing a comment. - * - * @param text The text of the comment (not including the `#`). - */ - case class CommentLine private[Consumer] ( - text: String, - indent: String - ) extends Line { - def contents = "# " + text - } - - /** A parsed TAP line containing a test plan. - * - * @param plan The [[com.iinteractive.test.Plan Plan]] that this line - * represents. - */ - case class PlanLine private[Consumer] ( - plan: Plan, - indent: String - ) extends Line { - def contents = { - val count = plan.plan - val comment = plan match { - case SkipAll(m) => " # SKIP " + m - case _ => "" - } - indent + "1.." + count + comment - } - } - - /** A parsed TAP line containing a test result. - * - * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] - * that this line represents. - */ - case class ResultLine private[Consumer] ( - result: TestResult, - indent: String - ) extends Line { - def contents = { - val success = (if (result.passed) "ok" else "not ok") + " " - val number = result.number + " " - val description = result.description match { - case "" => "" - case s => s + " " - } - val directive = result.directive.map { d => - d match { - case TodoDirective(m) => "# TODO " + m - case SkipDirective(m) => "# skip " + m - } - }.getOrElse("") - indent + success + number + description + directive - } - } - - private val commentRx = """^(\s*)#\s*(.*)""".r - private val planRx = """^(\s*)1..(\d+)\s*(?:# SKIP (.*))?""".r - private val resultRx = - """^(\s*)(not )?ok (\d+)\s*([^#]+)?(?:#\s*(?i:(skip|todo))\s+(.*))?""".r -} diff --git a/src/main/scala/org/perl8/test/tap/Parser.scala b/src/main/scala/org/perl8/test/tap/Parser.scala deleted file mode 100644 index 7bc44d3..0000000 --- a/src/main/scala/org/perl8/test/tap/Parser.scala +++ /dev/null @@ -1,232 +0,0 @@ -package com.iinteractive.test.tap - -import java.io.{ByteArrayInputStream,InputStream,OutputStream} -import scala.annotation.tailrec -import scala.io.Source -import scala.util.parsing.combinator._ -import scala.util.parsing.input.{Position,Reader} - -import com.iinteractive.test.Plan -import com.iinteractive.test.tap.Consumer._ - -/** This class parses a TAP stream. It can either parse it all at once (from a - * string), or it can be used as a streaming parser, where TAP events are - * emitted through a given callback. - */ -class Parser private ( - cb: TAPEvent => Unit, - indent: String -) { - /** Creates a parser instance. - * @param cb The event handler callback. It will be called after each - * meaningful line of TAP, with a - * [[com.iinteractive.test.tap.TAPEvent TAPEvent]] instance - * representing the event that was just parsed. - */ - def this (cb: TAPEvent => Unit = e => ()) = - this(cb, "") - - private def this (indent: String) = - this(e => (), indent) - - /** Parses TAP from an input stream. This variant will actually parse lines - * as they are available to read from the input stream, so this can be used - * as a streaming parser. - */ - def parse (input: InputStream): TAPResult = { - import parser._ - - cb(StartEvent) - tap(new LineReader(input)) match { - case Success(result, _) => { - cb(EndEvent(result)) - result - } - case failure: NoSuccess => throw new ParseException(failure.msg) - } - } - - /** Parses TAP contained in a string. This isn't useful for incremental - * parsing, because the entire input string must be created before - * parsing can begin. - */ - def parse (input: String): TAPResult = - parse(new ByteArrayInputStream(input.getBytes)) - - /** Parses TAP from an output stream. - * - * @todo Doesn't currently work as a streaming parser, since it just - * collects the entire output as a string and feeds it to the parser - * for strings. This could likely be improved. - */ - def parse (input: OutputStream): TAPResult = - parse(input.toString) - - private val parser = new TAPParser(cb, indent) - - private class TAPParser ( - cb: TAPEvent => Unit, - indent: String - ) extends Parsers { - type Elem = Line - - def tap: Parser[TAPResult] = - planFirst | planLast - - private def planFirst: Parser[TAPResult] = - plan ~ rep(result) ^^ { case plan ~ results => - new TAPResult(plan, results) - } - - private def planLast: Parser[TAPResult] = - rep(result) ~ plan ^^ { case results ~ plan => - new TAPResult(plan, results) - } - - private def plan: Parser[Plan] = - planLine ^^ { p => - cb(PlanEvent(p.plan)) - p.plan - } - - private def result: Parser[TestResult] = - simpleResult | subtestResult - - private def simpleResult: Parser[TestResult] = - resultLine ^^ { r => - cb(ResultEvent(r.result)) - r.result - } - - private def subtestResult: Parser[TestResult] = - subtest ~ simpleResult ^^ { case subtest ~ simpleResult => - new TestResult( - simpleResult.passed, - simpleResult.number, - simpleResult.description, - simpleResult.directive, - Some(subtest) - ) - } - - private def subtest: Parser[TAPResult] = - LineParser("subtest") { in => - // can't just return the result directly, because it's of a different - // type (the path dependent type associated with the new Parser - // instance we create here, rather than the path dependent type - // associated with this) - val subParser = new TAPParser( - e => (), - in.first.indent - ) - subParser.tap(in) match { - case subParser.Success(p, rest) => Success(p, rest) - case subParser.Failure(m, rest) => Failure(m, rest) - case subParser.Error(m, rest) => Error(m, rest) - } - } - - private def planLine: Parser[PlanLine] = LineParser("plan") { in => - val line = in.first - if (line.indent == indent) { - line match { - case p: PlanLine => - Success(p, in.rest) - case _ => - Failure("Plan line expected, but '" + line + "' found", in) - } - } - else { - Failure( - "Plan line expected, but " + - "'" + line + "' has incorrect indentation", - in - ) - } - } - - private def resultLine: Parser[ResultLine] = LineParser("result") { in => - val line = in.first - if (line.indent == indent) { - line match { - case p: ResultLine => - Success(p, in.rest) - case _ => - Failure("Result line expected, but '" + line + "' found", in) - } - } - else { - Failure( - "Result line expected, but " + - "'" + line + "' has incorrect indentation", - in - ) - } - } - - private def LineParser[T] (lineType: String)( - body: Input => ParseResult[T] - ): Parser[T] = Parser { in => - if (in.atEnd) { - Failure(lineType + " line expected, but end of input found", in) - } - else { - body(in) - } - } - } - - private class LineReader ( - in: Iterator[Char], - lineNum: Int - ) extends Reader[Line] { - def this (in: InputStream) = - this(Source.fromInputStream(in), 1) - - def atEnd: Boolean = - nextLine.isEmpty - - def first: Line = - nextLine.getOrElse(throw new RuntimeException("read from empty input")) - - lazy val pos = - new LinePosition(lineNum, nextLine.map(_.toString).getOrElse("")) - - lazy val rest: Reader[Line] = - new LineReader(remainingStream, lineNum + 1) - - private def nextLine: Option[Line] = - state._1 - - private def remainingStream: Iterator[Char] = - state._2 - - private lazy val state: (Option[Line], Iterator[Char]) = - readNextLine(in) - - @tailrec - private def readNextLine ( - stream: Iterator[Char] - ): (Option[Line], Iterator[Char]) = { - if (stream.hasNext) { - val (line, rest) = stream.span(_ != '\n') match { - case (l, r) => (parseLine(l.mkString), r.drop(1)) - } - line match { - case _: CommentLine => readNextLine(rest) - case other => (Some(other), rest) - } - } - else { - (None, in) - } - } - } - - private case class LinePosition ( - line: Int, - lineContents: String - ) extends Position { - def column: Int = 1 - } -} diff --git a/src/main/scala/org/perl8/test/tap/Producer.scala b/src/main/scala/org/perl8/test/tap/Producer.scala deleted file mode 100644 index 3ed8d7e..0000000 --- a/src/main/scala/org/perl8/test/tap/Producer.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.iinteractive.test.tap - -/** Contains functions for producing individual lines of TAP. */ -object Producer { - import com.iinteractive.test.Plan - - /** Returns a test result. - * - * @example `ok 4` - */ - def result (cond: Boolean, num: Int): String = - (if (cond) "ok " else "not ok ") + num - - /** Returns a test result that contains a description. - * - * @example `ok 28 - our test succeeded` - */ - def result (cond: Boolean, num: Int, desc: String): String = - result(cond, num) + " " + desc - - /** Returns a todo test result. - * - * @example `not ok 1 # TODO this doesn't work yet` - */ - def todoResult (cond: Boolean, num: Int, todo: String): String = - result(cond, num) + " # TODO " + todo - - /** Returns a todo test result that contains a description. - * - * @example `not ok 18 - test the feature # TODO can't figure this out` - */ - def todoResult (cond: Boolean, num: Int, desc: String, todo: String): String = - result(cond, num, desc) + " # TODO " + todo - - /** Returns a skipped test result with a reason. - * - * @example `ok 4 # skip this test won't run here` - */ - def skip (num: Int, reason: String): String = - "ok " + num + " # skip " + reason - - /** Returns a comment. - * - * @example `# this is a comment` - */ - def comment (message: String): String = - message.split("\n").map(m => "# " + m).mkString("\n") - - /** Returns a test plan. - * - * @example `1..5` ([[com.iinteractive.test.NumericPlan NumericPlan]]) - * @example `1..0 # SKIP don't run this test` ([[com.iinteractive.test.SkipAll SkipAll]]) - */ - def plan (plan: Plan): String = - plan.skipAll.map(m => "1..0 # SKIP " + m).getOrElse("1.." + plan.plan) - - /** Returns a bail out with a reason. - * - * @example `Bail out! Not supported on this platform.` - */ - def bailOut (message: String): String = - "Bail out! " + message -} diff --git a/src/main/scala/org/perl8/test/tap/TAPEvent.scala b/src/main/scala/org/perl8/test/tap/TAPEvent.scala deleted file mode 100644 index 1c2e88d..0000000 --- a/src/main/scala/org/perl8/test/tap/TAPEvent.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.iinteractive.test.tap - -import com.iinteractive.test.Plan - -/** An event emitted while parsing a TAP stream. */ -sealed trait TAPEvent - -/** The start of a TAP stream. */ -case object StartEvent extends TAPEvent - -/** The end of a TAP stream. - * @param result The [[com.iinteractive.test.tap.TAPResult TAPResult]] - * containing information about all of the tests which just - * finished running. This will be the same thing that is - * returned by the call to - * [[com.iinteractive.test.tap.Parser Parser]]'s `parse` - * method. - */ -case class EndEvent private[tap] (result: TAPResult) extends TAPEvent - -/** An individual test result. - * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] - * containing information about the corresponding test. - */ -case class ResultEvent private[tap] (result: TestResult) extends TAPEvent - -/** A test plan. - * @param plan The [[com.iinteractive.test.Plan Plan]] corresponding to the - * line that was parsed. - */ -case class PlanEvent private[tap] (plan: Plan) extends TAPEvent - -/** The start of a subtest (currently unused). */ -case object SubtestStartEvent extends TAPEvent - -/** The end of a subtest (currently unused). */ -case class SubtestEndEvent private[tap] (result: TestResult) extends TAPEvent - -/** A comment (currently unused). */ -case class CommentEvent private[tap] (text: String) extends TAPEvent diff --git a/src/main/scala/org/perl8/test/tap/TAPResult.scala b/src/main/scala/org/perl8/test/tap/TAPResult.scala deleted file mode 100644 index c3c4926..0000000 --- a/src/main/scala/org/perl8/test/tap/TAPResult.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.iinteractive.test.tap - -import com.iinteractive.test.{Plan,NumericPlan,SkipAll} - -/** The summarized results of a TAP stream. Contains the - * [[com.iinteractive.test.Plan Plan]] that was given, as well as a list of - * [[com.iinteractive.test.tap.TestResult TestResults]] corresponding to each - * of the tests in the stream. - * - * @param plan The [[com.iinteractive.test.Plan Plan]] from the TAP stream - * @param results The list of - * [[com.iinteractive.test.tap.TestResult TestResults]] from - * the TAP stream - */ -class TAPResult (val plan: Plan, val results: Seq[TestResult]) { - /** Returns true if the number of tests executed was compatible with the - * provided test plan. - */ - val matchesPlan = plan match { - case NumericPlan(n) => results.length == n - case _ => results.length == 0 - } - - /** Returns the number of test failures in the TAP stream. */ - val fails = results.count { r => - !r.passed && !r.directive.isDefined - } - - /** Returns true if all of the tests passed. */ - val testsPassed = fails == 0 - - /** Returns true if the TAP stream overall passed. - * - * Differs from `testsPassed` in that it also takes into account things - * like invalid plans. - */ - val success = plan match { - case SkipAll(_) => true - case _ => results.length > 0 && fails == 0 && matchesPlan - } - - /** Returns the exit code to use if running this test on its own. - * - * Success is indicated by 0, invalid TAP streams (such as incorrect plans) - * by 255, and other kinds of failures by the failure count. - */ - val exitCode = - if (success) { - 0 - } - else if (!matchesPlan || results.length == 0) { - 255 - } - else { - fails - } -} - -/** The result of a single test. - * - * @param passed True if the test passed - * @param number The test number in the TAP stream - * @param description The test description - * @param directive The [[com.iinteractive.test.tap.Directive Directive]] - * (either skip or todo) that was provided for this test, - * if any - * @param subtest The [[com.iinteractive.test.tap.TAPResult]] for the - * subtest that this test corresponds to, if any - */ -class TestResult ( - val passed: Boolean, - val number: Int, - val description: String, - val directive: Option[Directive], - val subtest: Option[TAPResult] -) - -/** A modifier associated with a test result. This is indicated by a `#` at - * the end of the result line, followed by the type of directive, and an - * optional message. - */ -sealed trait Directive { - val message: Option[String] -} - -/** A directive indicating that this test was skipped. */ -case class SkipDirective private[tap] ( - message: Option[String] -) extends Directive - -/** A directive indicating that this test is known to fail. */ -case class TodoDirective private[tap] ( - message: Option[String] -) extends Directive diff --git a/src/main/scala/org/perl8/test/tap/TestBuilder.scala b/src/main/scala/org/perl8/test/tap/TestBuilder.scala deleted file mode 100644 index 62fdf02..0000000 --- a/src/main/scala/org/perl8/test/tap/TestBuilder.scala +++ /dev/null @@ -1,196 +0,0 @@ -package com.iinteractive.test.tap - -import com.iinteractive.test._ - -/** This class provides a convenient yet low level API for generating TAP - * streams. Each instance of this class handles a single TAP stream, and - * keeps track of things like the current test number for you. All - * TAP-producing methods write the TAP lines to `Console.out` or - * `Console.err`, so you can override those (via `Console.withOut` or - * `Console.withErr`). - */ -class TestBuilder private ( - plan: Plan, - indent: String, - terminalInUse: Boolean -) { - plan match { - case NoPlan => () - case p => outLine(Producer.plan(p)) - } - - /** Creates a new builder instance, and emits the corresponding plan line, - * unless the plan is not given. - * - * @param plan [[com.iinteractive.test.Plan plan]] for this test. - * @param terminalInUse Whether this test is being run from a harness which - * will not just be writing directly to the output. - * This will make things written to `Console.err` have - * a newline prepended, so that they always start on - * an empty line. - */ - def this (plan: Plan = NoPlan, terminalInUse: Boolean = false) = - this(plan, "", terminalInUse) - - /** Create a new TestBuilder instance, to be used to run a subtest. This new - * instance will have all of its lines prefixed by an additional level of - * indentation. This instance will still need to have `doneTesting` - * called on it, and the result of the subtest will still need to be - * reported as a separate test result through `ok`. - */ - def cloneForSubtest (newPlan: Plan): TestBuilder = - new TestBuilder(newPlan, indent + " ", terminalInUse) - - /** Reports a single test result to `Console.out`. */ - def ok (test: Boolean) { - state.ok(test) - outLine(Producer.result(test, state.currentTest)) - } - - /** Reports a single test result with description to `Console.out`. */ - def ok (test: Boolean, description: String) { - state.ok(test) - outLine(Producer.result(test, state.currentTest, description)) - } - - /** Reports a single TODO test result to `Console.out`. */ - def todo (todo: String, test: Boolean) { - state.ok(true) - outLine(Producer.todoResult(test, state.currentTest, todo)) - } - - /** Reports a single TODO test result with description to `Console.out`. */ - def todo (todo: String, test: Boolean, description: String) { - state.ok(true) - outLine(Producer.todoResult(test, state.currentTest, description, todo)) - } - - /** Reports a single skipped test result to `Console.out`. */ - def skip (reason: String) { - state.ok(true) - outLine(Producer.skip(state.currentTest, reason)) - } - - /** Writes a comment line to `Console.err`. This will allow it to be - * visible in most summarizing harnesses (which consume and parse - * everything that goes to `Console.out`). - */ - def diag (message: String) { - errLine(Producer.comment(message)) - } - - /** Write a comment line to `Console.out`. This will typically only be - * visible in the raw TAP stream. - */ - def note (message: String) { - outLine(Producer.comment(message)) - } - - /** Abort the current test, with a message. */ - def bailOut (message: String) { - val bailOutMessage = Producer.bailOut(message) - outLine(bailOutMessage) - throw new BailOutException(bailOutMessage) - } - - /** Finalize the current builder instance. This writes the auto-calculated - * plan to `Console.out` if the plan type was `NoPlan` and reports a - * summary of the test results as a comment to `Console.err`. - * - * @return whether or not the test class as a whole passed. - */ - def doneTesting: Boolean = { - plan match { - case NumericPlan(_) => printErrors - case SkipAll(_) => () - case NoPlan => { - outLine(Producer.plan(state.currentTest)) - printErrors - } - } - state.isPassing - } - - /** The exit code to use, in harnesses that run a single test. Passing tests - * return 0, invalid tests (such as running a different number of tests - * than planned) return 255, and all others return the number of failed - * tests. - */ - def exitCode: Int = - if (state.isPassing) { - 0 - } - else if (!state.matchesPlan || state.currentTest == 0) { - 255 - } - else { - state.failCount - } - - private def printErrors { - if (!state.matchesPlan) { - val planCount = (plan match { - case NoPlan => state.currentTest - case p => p.plan - }) - val planned = planCount + " test" + (if (planCount > 1) "s" else "") - val ran = state.currentTest - diag("Looks like you planned " + planned + " but ran " + ran + ".") - } - - if (state.currentTest == 0) { - diag("No tests run!") - } - - if (state.failCount > 0) { - val count = state.failCount - val fails = count + " test" + (if (count > 1) "s" else "") - val total = - state.currentTest + (if (state.matchesPlan) "" else " run") - diag("Looks like you failed " + fails + " of " + total + ".") - } - } - - private val state = new TestState - - private def outLine (str: String) { - Console.out.println(withIndent(str)) - } - - private def errLine (str: String) { - if (terminalInUse) { - Console.err.print("\n") - } - Console.err.println(withIndent(str)) - } - - private def withIndent (str: String): String = - str.split("\n").map(s => indent + s).mkString("\n") - - private class TestState { - var passCount = 0 - var failCount = 0 - - def ok (cond: Boolean) { - if (cond) { - passCount += 1 - } - else { - failCount += 1 - } - } - - def currentTest: Int = - failCount + passCount - - def matchesPlan: Boolean = plan match { - case NumericPlan(p) => p.plan == currentTest - case _ => true - } - - def isPassing: Boolean = plan match { - case SkipAll(_) => true - case _ => currentTest > 0 && failCount == 0 && matchesPlan - } - } -} diff --git a/src/main/scala/org/perl8/test/tap/package.scala b/src/main/scala/org/perl8/test/tap/package.scala deleted file mode 100644 index 312f996..0000000 --- a/src/main/scala/org/perl8/test/tap/package.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.iinteractive.test - -/** Classes for TAP generation and parsing. */ -package object tap { - /** Exception representing an error during parsing. It is thrown when a TAP - * line isn't recognized. - */ - case class ParseException (message: String) extends RuntimeException(message) -} diff --git a/src/test/scala/com/iinteractive/test/ExtensionTest.scala b/src/test/scala/com/iinteractive/test/ExtensionTest.scala new file mode 100644 index 0000000..5c49f21 --- /dev/null +++ b/src/test/scala/com/iinteractive/test/ExtensionTest.scala @@ -0,0 +1,57 @@ +package com.iinteractive.test + +import java.io.ByteArrayOutputStream + +import com.iinteractive.test.tap.Parser + +trait NumberZero { this: TestMore => + def is_zero (i: Int, desc: String): Boolean = hideTestMethod { + is(i, 0, desc) + } +} + +trait NumberZeroWrapped extends NumberZero { this: TestMore => + def isZero (i: Int): Boolean = hideTestMethod { + is_zero(i, "the number is zero") + } +} + +class ExtensionTest extends TestMore { + val lineZero = Thread.currentThread.getStackTrace()(1).getLineNumber + 3 + def line (offset: Int) = lineZero + offset + + private class ExtensionTestTest extends TestMore with NumberZeroWrapped { + is_zero(0, "it's zero") + is_zero(1, "it's not zero") + isZero(0) + isZero(1) + } + + val out = new ByteArrayOutputStream + val exitCode = Console.withOut(out) { + Console.withErr(out) { + (new ExtensionTestTest).run + } + } + + is((new Parser).parse(out).exitCode, 2) + is(exitCode, 2) + + val tap = + "ok 1 - it's zero\n" + + "not ok 2 - it's not zero\n" + + "# Failed test 'it's not zero'\n" + + "# at ExtensionTest.scala line " + line(2) + ".\n" + + "# got: '1'\n" + + "# expected: '0'\n" + + "ok 3 - the number is zero\n" + + "not ok 4 - the number is zero\n" + + "# Failed test 'the number is zero'\n" + + "# at ExtensionTest.scala line " + line(4) + ".\n" + + "# got: '1'\n" + + "# expected: '0'\n" + + "1..4\n" + + "# Looks like you failed 2 tests of 4.\n" + + is(out.toString, tap) +} diff --git a/src/test/scala/com/iinteractive/test/ExternalTestTest.scala b/src/test/scala/com/iinteractive/test/ExternalTestTest.scala new file mode 100644 index 0000000..e830abe --- /dev/null +++ b/src/test/scala/com/iinteractive/test/ExternalTestTest.scala @@ -0,0 +1,3 @@ +package com.iinteractive.test + +class ExternalTestTest extends ExternalTest("perl", "perl/test.t") diff --git a/src/test/scala/com/iinteractive/test/PlanTest.scala b/src/test/scala/com/iinteractive/test/PlanTest.scala new file mode 100644 index 0000000..6d1690b --- /dev/null +++ b/src/test/scala/com/iinteractive/test/PlanTest.scala @@ -0,0 +1,29 @@ +package com.iinteractive.test + +import java.io.ByteArrayOutputStream + +import com.iinteractive.test.tap.Parser + +class PlanTest extends TestMore { + private class PlanTestTest extends TestMore(2) { + is(1, 1) + is(2, 2) + } + + val out = new ByteArrayOutputStream + val exitCode = Console.withOut(out) { + Console.withErr(out) { + (new PlanTestTest).run + } + } + + is((new Parser).parse(out).exitCode, 0) + is(exitCode, 0) + + val tap = + "1..2\n" + + "ok 1\n" + + "ok 2\n" + + is(out.toString, tap) +} diff --git a/src/test/scala/com/iinteractive/test/TestMoreTest.scala b/src/test/scala/com/iinteractive/test/TestMoreTest.scala new file mode 100644 index 0000000..ead9823 --- /dev/null +++ b/src/test/scala/com/iinteractive/test/TestMoreTest.scala @@ -0,0 +1,147 @@ +package com.iinteractive.test + +import java.io.ByteArrayOutputStream + +import com.iinteractive.test.tap.Parser + +class TestMoreTest extends TestMore { + val lineZero = Thread.currentThread.getStackTrace()(1).getLineNumber + 3 + def line (offset: Int) = lineZero + offset + + private class MyBasicTest extends TestMore { + diag("ok") + ok(1 == 1, "it works!") + ok(0 == 1, "it doesn't work!") + ok(1 == 1) + ok(0 == 1) + + diag("is") + is(1, 1, "it works!") + is(1, 0, "it doesn't work!") + is(1, 1) + is(1, 0) + + diag("isnt") + isnt(1, 0, "it works!") + isnt(1, 1, "it doesn't work!") + isnt(1, 0) + isnt(1, 1) + + diag("like") + like("foo", """foo""".r, "it works!") + like("foo", """bar""".r, "it doesn't work!") + like("foo", """foo""".r) + like("foo", """bar""".r) + + subtest("unlikes") { + diag("unlike") + unlike("foo", """bar""".r, "it works!") + unlike("foo", """foo""".r, "it doesn't work!") + unlike("foo", """bar""".r) + unlike("foo", """foo""".r) + } + + diag("pass") + pass("it works!") + pass + + skip(2, "don't do this yet") { + pass("skipped") + pass + } + + todo("not working yet") { + diag("fail") + fail("it doesn't work") + fail + } + } + + val out = new ByteArrayOutputStream + val exitCode = Console.withOut(out) { + Console.withErr(out) { + (new MyBasicTest).run + } + } + + is((new Parser).parse(out).exitCode, 9, "got the right plan") + is(exitCode, 9, "got the right plan") + + val expected = + "# ok\n" + + "ok 1 - it works!\n" + + "not ok 2 - it doesn't work!\n" + + "# Failed test 'it doesn't work!'\n" + + "# at TestMoreTest.scala line " + line(3) + ".\n" + + "ok 3\n" + + "not ok 4\n" + + "# Failed test at TestMoreTest.scala line " + line(5) + ".\n" + + "# is\n" + + "ok 5 - it works!\n" + + "not ok 6 - it doesn't work!\n" + + "# Failed test 'it doesn't work!'\n" + + "# at TestMoreTest.scala line " + line(9) + ".\n" + + "# got: '1'\n" + + "# expected: '0'\n" + + "ok 7\n" + + "not ok 8\n" + + "# Failed test at TestMoreTest.scala line " + line(11) + ".\n" + + "# got: '1'\n" + + "# expected: '0'\n" + + "# isnt\n" + + "ok 9 - it works!\n" + + "not ok 10 - it doesn't work!\n" + + "# Failed test 'it doesn't work!'\n" + + "# at TestMoreTest.scala line " + line(15) + ".\n" + + "# got: '1'\n" + + "# expected: anything else\n" + + "ok 11\n" + + "not ok 12\n" + + "# Failed test at TestMoreTest.scala line " + line(17) + ".\n" + + "# got: '1'\n" + + "# expected: anything else\n" + + "# like\n" + + "ok 13 - it works!\n" + + "not ok 14 - it doesn't work!\n" + + "# Failed test 'it doesn't work!'\n" + + "# at TestMoreTest.scala line " + line(21) + ".\n" + + "# 'foo'\n" + + "# doesn't match 'bar'\n" + + "ok 15\n" + + "not ok 16\n" + + "# Failed test at TestMoreTest.scala line " + line(23) + ".\n" + + "# 'foo'\n" + + "# doesn't match 'bar'\n" + + " # unlike\n" + + " ok 1 - it works!\n" + + " not ok 2 - it doesn't work!\n" + + " # Failed test 'it doesn't work!'\n" + + " # at TestMoreTest.scala line " + line(28) + ".\n" + + " # 'foo'\n" + + " # matches 'foo'\n" + + " ok 3\n" + + " not ok 4\n" + + " # Failed test at TestMoreTest.scala line " + line(30) + ".\n" + + " # 'foo'\n" + + " # matches 'foo'\n" + + " 1..4\n" + + " # Looks like you failed 2 tests of 4.\n" + + "not ok 17 - unlikes\n" + + "# Failed test 'unlikes'\n" + + "# at TestMoreTest.scala line " + line(25) + ".\n" + + "# pass\n" + + "ok 18 - it works!\n" + + "ok 19\n" + + "ok 20 # skip don't do this yet\n" + + "ok 21 # skip don't do this yet\n" + + "# fail\n" + + "not ok 22 - it doesn't work # TODO not working yet\n" + + "# Failed (TODO) test 'it doesn't work'\n" + + "# at TestMoreTest.scala line " + line(44) + ".\n" + + "not ok 23 # TODO not working yet\n" + + "# Failed (TODO) test at TestMoreTest.scala line " + line(45) + ".\n" + + "1..23\n" + + "# Looks like you failed 9 tests of 23.\n" + + is(out.toString, expected, "correct tap") +} diff --git a/src/test/scala/com/iinteractive/test/tap/ParserTest.scala b/src/test/scala/com/iinteractive/test/tap/ParserTest.scala new file mode 100644 index 0000000..bb871ee --- /dev/null +++ b/src/test/scala/com/iinteractive/test/tap/ParserTest.scala @@ -0,0 +1,127 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{TestMore,SkipAll,NumericPlan} + +class ParserTest extends TestMore { + subtest ("basic") { + val tap = + "1..1\n" + + "ok 1\n" + + val result = (new Parser).parse(tap) + is(result.plan, NumericPlan(1), "got the right plan") + is(result.results.map(_.passed), Seq(true), "got the right results") + } + + subtest ("skip all") { + val tap = + "1..0 # SKIP nope\n" + + val result = (new Parser).parse(tap) + is(result.plan, SkipAll("nope"), "got the right plan") + is(result.results, Nil, "got the right results") + } + + subtest ("more complicated") { + val tap = + "# starting...\n" + + "ok 1 - stuff\n" + + "not ok 2 - does this work?\n" + + "not ok 3 - eventually # TODO doesn't work yet\n" + + "# skipping some stuff\n" + + "ok 4 # skip don't do this yet\n" + + "# finished!\n" + + "1..4\n" + + "# Looks like you failed 1 test of 4.\n" + + val result = (new Parser).parse(tap) + is(result.plan, NumericPlan(4)) + is(result.results.map(_.passed), Seq(true, false, false, true)) + is(result.results.map(_.number), Seq(1, 2, 3, 4)) + is( + result.results.map(_.description), + Seq( + "- stuff", + "- does this work?", + "- eventually", + "" + ) + ) + + is( + result.results.map(_.directive), + Seq( + None, + None, + Some(TodoDirective(Some("doesn't work yet"))), + Some(SkipDirective(Some("don't do this yet"))) + ) + ) + } + + subtest ("subtests") { + val tap = + "ok 1 - not subtest\n" + + " ok 1 - passed\n" + + " not ok 2 - failed\n" + + " ok 3 - passed again\n" + + " 1..1\n" + + " ok 1 - sub-sub-test\n" + + " ok 4 - nested subtests\n" + + " 1..4\n" + + " # Looks like you failed 1 test of 4.\n" + + "not ok 2 - subtest\n" + + "1..2\n" + + "# Looks like you failed 1 test of 2.\n" + + val result = (new Parser).parse(tap) + is(result.plan, NumericPlan(2)) + is(result.results.map(_.passed), Seq(true, false)) + is(result.results.map(_.number), Seq(1, 2)) + is( + result.results.map(_.description), + Seq( + "- not subtest", + "- subtest" + ) + ) + is(result.results.map(_.directive), Seq(None, None)) + + is(result.results(0).subtest, None) + ok(result.results(1).subtest.isDefined) + + val subtest = result.results(1).subtest.get + is(subtest.plan, NumericPlan(4)) + is(subtest.results.map(_.passed), Seq(true, false, true, true)) + is(subtest.results.map(_.number), Seq(1, 2, 3, 4)) + is( + subtest.results.map(_.description), + Seq( + "- passed", + "- failed", + "- passed again", + "- nested subtests" + ) + ) + is(subtest.results.map(_.directive), Seq(None, None, None, None)) + + is(subtest.results(0).subtest, None) + is(subtest.results(1).subtest, None) + is(subtest.results(2).subtest, None) + ok(subtest.results(3).subtest.isDefined) + + val subsubtest = subtest.results(3).subtest.get + is(subsubtest.plan, NumericPlan(1)) + is(subsubtest.results.map(_.passed), Seq(true)) + is(subsubtest.results.map(_.number), Seq(1)) + is( + subsubtest.results.map(_.description), + Seq( + "- sub-sub-test" + ) + ) + is(subsubtest.results.map(_.directive), Seq(None)) + + is(subsubtest.results(0).subtest, None) + } +} diff --git a/src/test/scala/com/iinteractive/test/tap/TestBuilderTest.scala b/src/test/scala/com/iinteractive/test/tap/TestBuilderTest.scala new file mode 100644 index 0000000..bc873c8 --- /dev/null +++ b/src/test/scala/com/iinteractive/test/tap/TestBuilderTest.scala @@ -0,0 +1,241 @@ +package com.iinteractive.test.tap + +import java.io.ByteArrayOutputStream + +import com.iinteractive.test.{TestMore,SkipAll,BailOutException} + +class TestBuilderTest extends TestMore { + subtest ("ok") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder(4) + builder.ok(true, "test succeeded") + builder.ok(false, "test failed") + builder.ok(true) + builder.ok(false) + builder.doneTesting + } + } + + val expected = + "1..4\n" + + "ok 1 test succeeded\n" + + "not ok 2 test failed\n" + + "ok 3\n" + + "not ok 4\n" + + "# Looks like you failed 2 tests of 4.\n" + + is(output.toString, expected) + } + + subtest ("no plan") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true, "test succeeded") + builder.ok(false, "test failed") + builder.ok(true) + builder.ok(false) + builder.doneTesting + } + } + + val expected = + "ok 1 test succeeded\n" + + "not ok 2 test failed\n" + + "ok 3\n" + + "not ok 4\n" + + "1..4\n" + + "# Looks like you failed 2 tests of 4.\n" + + is(output.toString, expected) + } + + subtest ("empty") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.doneTesting + } + } + + val expected = + "1..0\n" + + "# No tests run!\n" + + is(output.toString, expected) + } + + subtest ("diag") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true, "the test passes") + builder.ok(false, "the test passes") + builder.diag("got false, expected true") + builder.ok(true) + builder.diag("ending\nnow") + builder.doneTesting + } + } + + val expected = + "ok 1 the test passes\n" + + "not ok 2 the test passes\n" + + "# got false, expected true\n" + + "ok 3\n" + + "# ending\n" + + "# now\n" + + "1..3\n" + + "# Looks like you failed 1 test of 3.\n" + + is(output.toString, expected) + } + + subtest ("is passing") { + val output = new ByteArrayOutputStream + val oldOut = Console.out + val oldErr = Console.err + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.doneTesting + } + }, + false + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.doneTesting + } + }, + true + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.ok(false) + builder.doneTesting + } + }, + false + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.ok(false) + builder.ok(true) + builder.doneTesting + } + }, + false + ) + } + + subtest ("bail out") { + val output = new ByteArrayOutputStream + val oldOut = Console.out + val oldErr = Console.err + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + try { + builder.bailOut("oh no!") + Console.withOut(oldOut) { + Console.withErr(oldErr) { + fail + } + } + } + catch { + case e: BailOutException => Console.withOut(oldOut) { + Console.withErr(oldErr) { + is(e.message, "Bail out! oh no!") + } + } + case _: Throwable => Console.withOut(oldOut) { + Console.withErr(oldErr) { + fail + } + } + } + } + } + + val expected = + "ok 1\n" + + "Bail out! oh no!\n" + + is(output.toString, expected) + } + + subtest ("skip all") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder(SkipAll("foo bar")) + } + } + + val expected = + "1..0 # SKIP foo bar\n" + + is(output.toString, expected) + } + + subtest ("skip") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(false) + builder.skip("not now") + builder.doneTesting + } + } + + val expected = + "not ok 1\n" + + "ok 2 # skip not now\n" + + "1..2\n" + + "# Looks like you failed 1 test of 2.\n" + + is(output.toString, expected) + } + + subtest ("todo") { + val output = new ByteArrayOutputStream + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.todo("not working yet", false, "do a thing") + builder.todo("is it?", true) + builder.doneTesting + } + } + + val expected = + "not ok 1 do a thing # TODO not working yet\n" + + "ok 2 # TODO is it?\n" + + "1..2\n" + + is(output.toString, expected) + } +} diff --git a/src/test/scala/org/perl8/test/ExtensionTest.scala b/src/test/scala/org/perl8/test/ExtensionTest.scala deleted file mode 100644 index 5c49f21..0000000 --- a/src/test/scala/org/perl8/test/ExtensionTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.iinteractive.test - -import java.io.ByteArrayOutputStream - -import com.iinteractive.test.tap.Parser - -trait NumberZero { this: TestMore => - def is_zero (i: Int, desc: String): Boolean = hideTestMethod { - is(i, 0, desc) - } -} - -trait NumberZeroWrapped extends NumberZero { this: TestMore => - def isZero (i: Int): Boolean = hideTestMethod { - is_zero(i, "the number is zero") - } -} - -class ExtensionTest extends TestMore { - val lineZero = Thread.currentThread.getStackTrace()(1).getLineNumber + 3 - def line (offset: Int) = lineZero + offset - - private class ExtensionTestTest extends TestMore with NumberZeroWrapped { - is_zero(0, "it's zero") - is_zero(1, "it's not zero") - isZero(0) - isZero(1) - } - - val out = new ByteArrayOutputStream - val exitCode = Console.withOut(out) { - Console.withErr(out) { - (new ExtensionTestTest).run - } - } - - is((new Parser).parse(out).exitCode, 2) - is(exitCode, 2) - - val tap = - "ok 1 - it's zero\n" + - "not ok 2 - it's not zero\n" + - "# Failed test 'it's not zero'\n" + - "# at ExtensionTest.scala line " + line(2) + ".\n" + - "# got: '1'\n" + - "# expected: '0'\n" + - "ok 3 - the number is zero\n" + - "not ok 4 - the number is zero\n" + - "# Failed test 'the number is zero'\n" + - "# at ExtensionTest.scala line " + line(4) + ".\n" + - "# got: '1'\n" + - "# expected: '0'\n" + - "1..4\n" + - "# Looks like you failed 2 tests of 4.\n" - - is(out.toString, tap) -} diff --git a/src/test/scala/org/perl8/test/ExternalTestTest.scala b/src/test/scala/org/perl8/test/ExternalTestTest.scala deleted file mode 100644 index e830abe..0000000 --- a/src/test/scala/org/perl8/test/ExternalTestTest.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.iinteractive.test - -class ExternalTestTest extends ExternalTest("perl", "perl/test.t") diff --git a/src/test/scala/org/perl8/test/PlanTest.scala b/src/test/scala/org/perl8/test/PlanTest.scala deleted file mode 100644 index 6d1690b..0000000 --- a/src/test/scala/org/perl8/test/PlanTest.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.iinteractive.test - -import java.io.ByteArrayOutputStream - -import com.iinteractive.test.tap.Parser - -class PlanTest extends TestMore { - private class PlanTestTest extends TestMore(2) { - is(1, 1) - is(2, 2) - } - - val out = new ByteArrayOutputStream - val exitCode = Console.withOut(out) { - Console.withErr(out) { - (new PlanTestTest).run - } - } - - is((new Parser).parse(out).exitCode, 0) - is(exitCode, 0) - - val tap = - "1..2\n" + - "ok 1\n" + - "ok 2\n" - - is(out.toString, tap) -} diff --git a/src/test/scala/org/perl8/test/TestMoreTest.scala b/src/test/scala/org/perl8/test/TestMoreTest.scala deleted file mode 100644 index ead9823..0000000 --- a/src/test/scala/org/perl8/test/TestMoreTest.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.iinteractive.test - -import java.io.ByteArrayOutputStream - -import com.iinteractive.test.tap.Parser - -class TestMoreTest extends TestMore { - val lineZero = Thread.currentThread.getStackTrace()(1).getLineNumber + 3 - def line (offset: Int) = lineZero + offset - - private class MyBasicTest extends TestMore { - diag("ok") - ok(1 == 1, "it works!") - ok(0 == 1, "it doesn't work!") - ok(1 == 1) - ok(0 == 1) - - diag("is") - is(1, 1, "it works!") - is(1, 0, "it doesn't work!") - is(1, 1) - is(1, 0) - - diag("isnt") - isnt(1, 0, "it works!") - isnt(1, 1, "it doesn't work!") - isnt(1, 0) - isnt(1, 1) - - diag("like") - like("foo", """foo""".r, "it works!") - like("foo", """bar""".r, "it doesn't work!") - like("foo", """foo""".r) - like("foo", """bar""".r) - - subtest("unlikes") { - diag("unlike") - unlike("foo", """bar""".r, "it works!") - unlike("foo", """foo""".r, "it doesn't work!") - unlike("foo", """bar""".r) - unlike("foo", """foo""".r) - } - - diag("pass") - pass("it works!") - pass - - skip(2, "don't do this yet") { - pass("skipped") - pass - } - - todo("not working yet") { - diag("fail") - fail("it doesn't work") - fail - } - } - - val out = new ByteArrayOutputStream - val exitCode = Console.withOut(out) { - Console.withErr(out) { - (new MyBasicTest).run - } - } - - is((new Parser).parse(out).exitCode, 9, "got the right plan") - is(exitCode, 9, "got the right plan") - - val expected = - "# ok\n" + - "ok 1 - it works!\n" + - "not ok 2 - it doesn't work!\n" + - "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line " + line(3) + ".\n" + - "ok 3\n" + - "not ok 4\n" + - "# Failed test at TestMoreTest.scala line " + line(5) + ".\n" + - "# is\n" + - "ok 5 - it works!\n" + - "not ok 6 - it doesn't work!\n" + - "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line " + line(9) + ".\n" + - "# got: '1'\n" + - "# expected: '0'\n" + - "ok 7\n" + - "not ok 8\n" + - "# Failed test at TestMoreTest.scala line " + line(11) + ".\n" + - "# got: '1'\n" + - "# expected: '0'\n" + - "# isnt\n" + - "ok 9 - it works!\n" + - "not ok 10 - it doesn't work!\n" + - "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line " + line(15) + ".\n" + - "# got: '1'\n" + - "# expected: anything else\n" + - "ok 11\n" + - "not ok 12\n" + - "# Failed test at TestMoreTest.scala line " + line(17) + ".\n" + - "# got: '1'\n" + - "# expected: anything else\n" + - "# like\n" + - "ok 13 - it works!\n" + - "not ok 14 - it doesn't work!\n" + - "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line " + line(21) + ".\n" + - "# 'foo'\n" + - "# doesn't match 'bar'\n" + - "ok 15\n" + - "not ok 16\n" + - "# Failed test at TestMoreTest.scala line " + line(23) + ".\n" + - "# 'foo'\n" + - "# doesn't match 'bar'\n" + - " # unlike\n" + - " ok 1 - it works!\n" + - " not ok 2 - it doesn't work!\n" + - " # Failed test 'it doesn't work!'\n" + - " # at TestMoreTest.scala line " + line(28) + ".\n" + - " # 'foo'\n" + - " # matches 'foo'\n" + - " ok 3\n" + - " not ok 4\n" + - " # Failed test at TestMoreTest.scala line " + line(30) + ".\n" + - " # 'foo'\n" + - " # matches 'foo'\n" + - " 1..4\n" + - " # Looks like you failed 2 tests of 4.\n" + - "not ok 17 - unlikes\n" + - "# Failed test 'unlikes'\n" + - "# at TestMoreTest.scala line " + line(25) + ".\n" + - "# pass\n" + - "ok 18 - it works!\n" + - "ok 19\n" + - "ok 20 # skip don't do this yet\n" + - "ok 21 # skip don't do this yet\n" + - "# fail\n" + - "not ok 22 - it doesn't work # TODO not working yet\n" + - "# Failed (TODO) test 'it doesn't work'\n" + - "# at TestMoreTest.scala line " + line(44) + ".\n" + - "not ok 23 # TODO not working yet\n" + - "# Failed (TODO) test at TestMoreTest.scala line " + line(45) + ".\n" + - "1..23\n" + - "# Looks like you failed 9 tests of 23.\n" - - is(out.toString, expected, "correct tap") -} diff --git a/src/test/scala/org/perl8/test/tap/ParserTest.scala b/src/test/scala/org/perl8/test/tap/ParserTest.scala deleted file mode 100644 index bb871ee..0000000 --- a/src/test/scala/org/perl8/test/tap/ParserTest.scala +++ /dev/null @@ -1,127 +0,0 @@ -package com.iinteractive.test.tap - -import com.iinteractive.test.{TestMore,SkipAll,NumericPlan} - -class ParserTest extends TestMore { - subtest ("basic") { - val tap = - "1..1\n" + - "ok 1\n" - - val result = (new Parser).parse(tap) - is(result.plan, NumericPlan(1), "got the right plan") - is(result.results.map(_.passed), Seq(true), "got the right results") - } - - subtest ("skip all") { - val tap = - "1..0 # SKIP nope\n" - - val result = (new Parser).parse(tap) - is(result.plan, SkipAll("nope"), "got the right plan") - is(result.results, Nil, "got the right results") - } - - subtest ("more complicated") { - val tap = - "# starting...\n" + - "ok 1 - stuff\n" + - "not ok 2 - does this work?\n" + - "not ok 3 - eventually # TODO doesn't work yet\n" + - "# skipping some stuff\n" + - "ok 4 # skip don't do this yet\n" + - "# finished!\n" + - "1..4\n" + - "# Looks like you failed 1 test of 4.\n" - - val result = (new Parser).parse(tap) - is(result.plan, NumericPlan(4)) - is(result.results.map(_.passed), Seq(true, false, false, true)) - is(result.results.map(_.number), Seq(1, 2, 3, 4)) - is( - result.results.map(_.description), - Seq( - "- stuff", - "- does this work?", - "- eventually", - "" - ) - ) - - is( - result.results.map(_.directive), - Seq( - None, - None, - Some(TodoDirective(Some("doesn't work yet"))), - Some(SkipDirective(Some("don't do this yet"))) - ) - ) - } - - subtest ("subtests") { - val tap = - "ok 1 - not subtest\n" + - " ok 1 - passed\n" + - " not ok 2 - failed\n" + - " ok 3 - passed again\n" + - " 1..1\n" + - " ok 1 - sub-sub-test\n" + - " ok 4 - nested subtests\n" + - " 1..4\n" + - " # Looks like you failed 1 test of 4.\n" + - "not ok 2 - subtest\n" + - "1..2\n" + - "# Looks like you failed 1 test of 2.\n" - - val result = (new Parser).parse(tap) - is(result.plan, NumericPlan(2)) - is(result.results.map(_.passed), Seq(true, false)) - is(result.results.map(_.number), Seq(1, 2)) - is( - result.results.map(_.description), - Seq( - "- not subtest", - "- subtest" - ) - ) - is(result.results.map(_.directive), Seq(None, None)) - - is(result.results(0).subtest, None) - ok(result.results(1).subtest.isDefined) - - val subtest = result.results(1).subtest.get - is(subtest.plan, NumericPlan(4)) - is(subtest.results.map(_.passed), Seq(true, false, true, true)) - is(subtest.results.map(_.number), Seq(1, 2, 3, 4)) - is( - subtest.results.map(_.description), - Seq( - "- passed", - "- failed", - "- passed again", - "- nested subtests" - ) - ) - is(subtest.results.map(_.directive), Seq(None, None, None, None)) - - is(subtest.results(0).subtest, None) - is(subtest.results(1).subtest, None) - is(subtest.results(2).subtest, None) - ok(subtest.results(3).subtest.isDefined) - - val subsubtest = subtest.results(3).subtest.get - is(subsubtest.plan, NumericPlan(1)) - is(subsubtest.results.map(_.passed), Seq(true)) - is(subsubtest.results.map(_.number), Seq(1)) - is( - subsubtest.results.map(_.description), - Seq( - "- sub-sub-test" - ) - ) - is(subsubtest.results.map(_.directive), Seq(None)) - - is(subsubtest.results(0).subtest, None) - } -} diff --git a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala b/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala deleted file mode 100644 index bc873c8..0000000 --- a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala +++ /dev/null @@ -1,241 +0,0 @@ -package com.iinteractive.test.tap - -import java.io.ByteArrayOutputStream - -import com.iinteractive.test.{TestMore,SkipAll,BailOutException} - -class TestBuilderTest extends TestMore { - subtest ("ok") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder(4) - builder.ok(true, "test succeeded") - builder.ok(false, "test failed") - builder.ok(true) - builder.ok(false) - builder.doneTesting - } - } - - val expected = - "1..4\n" + - "ok 1 test succeeded\n" + - "not ok 2 test failed\n" + - "ok 3\n" + - "not ok 4\n" + - "# Looks like you failed 2 tests of 4.\n" - - is(output.toString, expected) - } - - subtest ("no plan") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true, "test succeeded") - builder.ok(false, "test failed") - builder.ok(true) - builder.ok(false) - builder.doneTesting - } - } - - val expected = - "ok 1 test succeeded\n" + - "not ok 2 test failed\n" + - "ok 3\n" + - "not ok 4\n" + - "1..4\n" + - "# Looks like you failed 2 tests of 4.\n" - - is(output.toString, expected) - } - - subtest ("empty") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.doneTesting - } - } - - val expected = - "1..0\n" + - "# No tests run!\n" - - is(output.toString, expected) - } - - subtest ("diag") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true, "the test passes") - builder.ok(false, "the test passes") - builder.diag("got false, expected true") - builder.ok(true) - builder.diag("ending\nnow") - builder.doneTesting - } - } - - val expected = - "ok 1 the test passes\n" + - "not ok 2 the test passes\n" + - "# got false, expected true\n" + - "ok 3\n" + - "# ending\n" + - "# now\n" + - "1..3\n" + - "# Looks like you failed 1 test of 3.\n" - - is(output.toString, expected) - } - - subtest ("is passing") { - val output = new ByteArrayOutputStream - val oldOut = Console.out - val oldErr = Console.err - - is( - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.doneTesting - } - }, - false - ) - - is( - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true) - builder.doneTesting - } - }, - true - ) - - is( - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true) - builder.ok(false) - builder.doneTesting - } - }, - false - ) - - is( - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true) - builder.ok(false) - builder.ok(true) - builder.doneTesting - } - }, - false - ) - } - - subtest ("bail out") { - val output = new ByteArrayOutputStream - val oldOut = Console.out - val oldErr = Console.err - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(true) - try { - builder.bailOut("oh no!") - Console.withOut(oldOut) { - Console.withErr(oldErr) { - fail - } - } - } - catch { - case e: BailOutException => Console.withOut(oldOut) { - Console.withErr(oldErr) { - is(e.message, "Bail out! oh no!") - } - } - case _: Throwable => Console.withOut(oldOut) { - Console.withErr(oldErr) { - fail - } - } - } - } - } - - val expected = - "ok 1\n" + - "Bail out! oh no!\n" - - is(output.toString, expected) - } - - subtest ("skip all") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder(SkipAll("foo bar")) - } - } - - val expected = - "1..0 # SKIP foo bar\n" - - is(output.toString, expected) - } - - subtest ("skip") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.ok(false) - builder.skip("not now") - builder.doneTesting - } - } - - val expected = - "not ok 1\n" + - "ok 2 # skip not now\n" + - "1..2\n" + - "# Looks like you failed 1 test of 2.\n" - - is(output.toString, expected) - } - - subtest ("todo") { - val output = new ByteArrayOutputStream - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - builder.todo("not working yet", false, "do a thing") - builder.todo("is it?", true) - builder.doneTesting - } - } - - val expected = - "not ok 1 do a thing # TODO not working yet\n" + - "ok 2 # TODO is it?\n" + - "1..2\n" - - is(output.toString, expected) - } -} -- cgit v1.2.3