tests shouldn't produce anything other than tap

the harness can parse the tap to figure out whatever it needs from that

11 files changed, 137 insertions, 90 deletions

diff --git a/src/main/scala/org/perl8/test/Test.scala b/src/main/scala/org/perl8/test/Test.scala
index 93933dd..cc79df8 100644
--- a/src/main/scala/org/perl8/test/Test.scala
+++ b/src/main/scala/org/perl8/test/Test.scala
@@ -6,5 +6,5 @@ trait Test {
   /** Runs the test. The TAP stream will be written to Console.out and
     * Console.err, so you can swap these out as required in order to parse it.
     */
-  def run (): Int
+  def run (): Unit
 }
diff --git a/src/main/scala/org/perl8/test/TestMore.scala b/src/main/scala/org/perl8/test/TestMore.scala
index 487b023..2253d31 100644
--- a/src/main/scala/org/perl8/test/TestMore.scala
+++ b/src/main/scala/org/perl8/test/TestMore.scala
@@ -16,22 +16,13 @@ class TestMore (plan: Option[Plan] = None) extends Test with DelayedInit {
     testBody = () => body
   }
 
-  def run (): Int = {
+  def run () {
     if (testBody == null) {
       delayedInit { }
     }
 
     testBody()
     builder.doneTesting
-    if (builder.isPassing) {
-      0
-    }
-    else if (builder.failedTests == 0) {
-      255
-    }
-    else {
-      builder.failedTests
-    }
   }
 
   def ok (cond: Boolean, desc: Message = NoMessage): Boolean = {
@@ -125,7 +116,6 @@ class TestMore (plan: Option[Plan] = None) extends Test with DelayedInit {
       )
       body
       builder.doneTesting
-      builder.isPassing
     }
     finally {
       builder = oldBuilder
diff --git a/src/main/scala/org/perl8/test/Utils.scala b/src/main/scala/org/perl8/test/Utils.scala
index 5135688..31a66b1 100644
--- a/src/main/scala/org/perl8/test/Utils.scala
+++ b/src/main/scala/org/perl8/test/Utils.scala
@@ -10,13 +10,10 @@ object Utils {
   }
 
   case class NumericPlan (
-    override val plan: Int,
-    override val message: Option[String] = None
+    override val plan: Int
   ) extends Plan {
     override val skipAll = false
-
-    def this (plan: Int, message: String) =
-      this(plan, Some(message))
+    override val message = None
   }
 
   case class SkipAll (
diff --git a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala
index 23e5244..460059a 100644
--- a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala
+++ b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala
@@ -10,12 +10,12 @@ import Utils._
 class SummaryReporter extends MultiTestReporter {
   def run (testNames: Seq[String]): Int = {
     val results = runTests(testNames)
-    val success = results.values.map(_._1).forall(_ == 0)
+    val success = results.values.forall(_.success)
     printTestSummary(success, results)
     if (success) 0 else 1
   }
 
-  def runTests (testNames: Seq[String]): Map[String, (Int, TAPResult)] = {
+  def runTests (testNames: Seq[String]): Map[String, TAPResult] = {
     val maxLength = testNames.map(_.length).max
 
     testNames.map { name =>
@@ -23,12 +23,12 @@ class SummaryReporter extends MultiTestReporter {
 
       val out = new ByteArrayOutputStream
       val test = newInstance[Test](name)
-      val exitCode = Console.withOut(out) {
+      Console.withOut(out) {
         test.run
       }
       val result = tap.Consumer.parse(out)
 
-      if (exitCode == 0) {
+      if (result.success) {
         println("ok")
       }
       else {
@@ -37,17 +37,17 @@ class SummaryReporter extends MultiTestReporter {
           !t.passed && !t.directive.isDefined
         }
 
-        println("Dubious, test returned " + exitCode)
+        println("Dubious, test returned " + result.exitCode)
         println("Failed " + failed + "/" + results + " subtests")
       }
 
-      name -> (exitCode, result)
+      name -> result
     }.toMap
   }
 
   def printTestSummary (
     success: Boolean,
-    results: Map[String, (Int, TAPResult)]
+    results: Map[String, TAPResult]
   ) {
     printSuccess(success)
     printShortSummary(results)
@@ -61,16 +61,14 @@ class SummaryReporter extends MultiTestReporter {
     }
   }
 
-  private def printShortSummary (results: Map[String, (Int, TAPResult)]) {
+  private def printShortSummary (results: Map[String, TAPResult]) {
     val files = results.size
-    val tests = results.values.map(_._2.results.length).sum
+    val tests = results.values.map(_.results.length).sum
     println("Files=" + files + ", Tests=" + tests)
   }
 
-  private def printLongSummary (results: Map[String, (Int, TAPResult)]) {
-    val resultMap = results.map { case (s, r) => s -> r._2 }
-
-    val todoSucceeded = resultMap.mapValues { r =>
+  private def printLongSummary (results: Map[String, TAPResult]) {
+    val todoSucceeded = results.mapValues { r =>
       r.results.filter { t =>
         t.directive match {
           case Some(TodoDirective(_)) => t.passed
@@ -79,7 +77,7 @@ class SummaryReporter extends MultiTestReporter {
       }
     }.filter(_._2.length > 0)
 
-    val testsFailed = resultMap.mapValues { r =>
+    val testsFailed = results.mapValues { r =>
       r.results.filter { t =>
         t.directive match {
           case None => !t.passed
@@ -98,7 +96,7 @@ class SummaryReporter extends MultiTestReporter {
       val maxLength = testNames.map(_.length).max
 
       for (name <- testNames) {
-        val result = resultMap(name)
+        val result = results(name)
 
         println(
           name + (" " * (maxLength - name.length)) + "               " +
@@ -122,7 +120,7 @@ class SummaryReporter extends MultiTestReporter {
           )
         }
 
-        val exitCode = results(name)._1
+        val exitCode = results(name).exitCode
         if (exitCode != 0) {
           println("  Non-zero exit status: " + exitCode)
         }
@@ -132,7 +130,7 @@ class SummaryReporter extends MultiTestReporter {
 
   private def printPassFail (
     success: Boolean,
-    results: Map[String, (Int, TAPResult)]
+    results: Map[String, TAPResult]
   ) {
     if (success) {
       println("Result: PASS")
@@ -140,7 +138,7 @@ class SummaryReporter extends MultiTestReporter {
     else {
       println("Result: FAIL")
 
-      val testResults = results.values.map(_._2)
+      val testResults = results.values
 
       val testsFailed = testResults.map { r =>
         r.results.count { t =>
diff --git a/src/main/scala/org/perl8/test/harness/TAPReporter.scala b/src/main/scala/org/perl8/test/harness/TAPReporter.scala
index e478880..574dd39 100644
--- a/src/main/scala/org/perl8/test/harness/TAPReporter.scala
+++ b/src/main/scala/org/perl8/test/harness/TAPReporter.scala
@@ -1,9 +1,25 @@
 package org.perl8.test.harness
 
+import java.io.ByteArrayOutputStream
+
+import org.perl8.test.tap
 import org.perl8.test.Test
 import Utils._
 
 class TAPReporter extends Reporter {
-  def run (testName: String): Int =
-    newInstance[Test](testName).run
+  def run (testName: String): Int = {
+    val out = new ByteArrayOutputStream
+    Console.withOut(out) {
+      Console.withErr(out) {
+        newInstance[Test](testName).run
+      }
+    }
+
+    // XXX this is wrong: it sends everything to stdout
+    // need to write a tee-like outputstream to fix it
+    print(out)
+
+    val result = tap.Consumer.parse(out)
+    result.exitCode
+  }
 }
diff --git a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala b/src/main/scala/org/perl8/test/sbt/SBTReporter.scala
index 6307f55..2f47af2 100644
--- a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala
+++ b/src/main/scala/org/perl8/test/sbt/SBTReporter.scala
@@ -41,14 +41,14 @@ class SBTReporter (
       eventHandler.handle(event)
     }
 
-    if (result.results.exists { r => !r.passed && !r.directive.isDefined }) {
-      logError(testName + " failed.")
-      1
-    }
-    else {
+    if (result.success) {
       logInfo(testName + " succeeded.")
       0
     }
+    else {
+      logError(testName + " failed.")
+      1
+    }
   }
 
   private def logDebug (msg: String) {
diff --git a/src/main/scala/org/perl8/test/tap/Consumer.scala b/src/main/scala/org/perl8/test/tap/Consumer.scala
index 7a0f7aa..2c542f1 100644
--- a/src/main/scala/org/perl8/test/tap/Consumer.scala
+++ b/src/main/scala/org/perl8/test/tap/Consumer.scala
@@ -135,7 +135,32 @@ case class TestResult (
   val subtest:     Option[TAPResult]
 )
 
-class TAPResult (val plan: Plan, val results: Seq[TestResult])
+class TAPResult (val plan: Plan, val results: Seq[TestResult]) {
+  val correctPlan = plan match {
+    case NumericPlan(n) => results.length == n
+    case SkipAll(_)     => results.length == 0
+  }
+
+  val fails = results.count { r =>
+    !r.passed && !r.directive.isDefined
+  }
+
+  val testsPassed = fails == 0
+
+  val success =
+    correctPlan && testsPassed
+
+  val exitCode =
+    if (success) {
+      0
+    }
+    else if (!correctPlan) {
+      255
+    }
+    else {
+      fails
+    }
+}
 
 case class ParseException (
   val message: String
diff --git a/src/main/scala/org/perl8/test/tap/TestBuilder.scala b/src/main/scala/org/perl8/test/tap/TestBuilder.scala
index 203080c..b1d8579 100644
--- a/src/main/scala/org/perl8/test/tap/TestBuilder.scala
+++ b/src/main/scala/org/perl8/test/tap/TestBuilder.scala
@@ -51,13 +51,13 @@ class TestBuilder (
     throw new BailOutException(message.getOrElse(""))
   }
 
-  def doneTesting () {
+  def doneTesting (): Boolean = {
     plan match {
       case None => outLine(Producer.plan(state.currentTest - 1))
       case _    => ()
     }
 
-    if (!isPassing) {
+    if (!state.isPassing) {
       if (!state.matchesPlan) {
         val planCount = (plan match {
           case Some(p) => p.plan
@@ -80,10 +80,9 @@ class TestBuilder (
         diag("Looks like you failed " + fails + " of " + total + ".")
       }
     }
-  }
 
-  def isPassing: Boolean =
     state.isPassing
+  }
 
   def failedTests: Int =
     state.failCount
diff --git a/src/test/scala/org/perl8/test/ExtensionTest.scala b/src/test/scala/org/perl8/test/ExtensionTest.scala
index e616d3e..6455e16 100644
--- a/src/test/scala/org/perl8/test/ExtensionTest.scala
+++ b/src/test/scala/org/perl8/test/ExtensionTest.scala
@@ -2,6 +2,8 @@ package org.perl8.test
 
 import java.io.ByteArrayOutputStream
 
+import org.perl8.test.tap.Consumer
+
 trait NumberZero { this: TestMore =>
   def is_zero (i: Int, desc: String): Boolean = {
     withLevel(1) {
@@ -27,25 +29,25 @@ class ExtensionTest extends TestMore {
   }
 
   val out = new ByteArrayOutputStream
-  val exitCode = Console.withOut(out) {
+  Console.withOut(out) {
     Console.withErr(out) {
       (new ExtensionTestTest).run
     }
   }
 
-  is(exitCode, 2)
+  is(Consumer.parse(out).exitCode, 2)
 
   val tap =
     "ok 1 - it's zero\n" +
     "not ok 2 - it's not zero\n" +
     "#   Failed test 'it's not zero'\n" +
-    "#   at ExtensionTest.scala line 24.\n" +
+    "#   at ExtensionTest.scala line 26.\n" +
     "#          got: '1'\n" +
     "#     expected: '0'\n" +
     "ok 3 - the number is zero\n" +
     "not ok 4 - the number is zero\n" +
     "#   Failed test 'the number is zero'\n" +
-    "#   at ExtensionTest.scala line 26.\n" +
+    "#   at ExtensionTest.scala line 28.\n" +
     "#          got: '1'\n" +
     "#     expected: '0'\n" +
     "1..4\n" +
diff --git a/src/test/scala/org/perl8/test/TestMoreTest.scala b/src/test/scala/org/perl8/test/TestMoreTest.scala
index 61bacad..a72c18b 100644
--- a/src/test/scala/org/perl8/test/TestMoreTest.scala
+++ b/src/test/scala/org/perl8/test/TestMoreTest.scala
@@ -2,6 +2,8 @@ package org.perl8.test
 
 import java.io.ByteArrayOutputStream
 
+import org.perl8.test.tap.Consumer
+
 class TestMoreTest extends TestMore {
   private object OutputContainer {
     val output = new ByteArrayOutputStream
@@ -56,76 +58,76 @@ class TestMoreTest extends TestMore {
     }
   }
 
-  val exitCode = Console.withOut(OutputContainer.output) {
+  Console.withOut(OutputContainer.output) {
     Console.withErr(OutputContainer.output) {
       (new MyBasicTest).run
     }
   }
 
-  is(exitCode, 9, "got the right plan")
+  is(Consumer.parse(OutputContainer.output).exitCode, 9, "got the right plan")
 
   val expected =
     "# ok\n" +
     "ok 1 - it works!\n" +
     "not ok 2 - it doesn't work!\n" +
     "#   Failed test 'it doesn't work!'\n" +
-    "#   at TestMoreTest.scala line 13.\n" +
+    "#   at TestMoreTest.scala line 15.\n" +
     "ok 3\n" +
     "not ok 4\n" +
-    "#   Failed test at TestMoreTest.scala line 15.\n" +
+    "#   Failed test at TestMoreTest.scala line 17.\n" +
     "# is\n" +
     "ok 5 - it works!\n" +
     "not ok 6 - it doesn't work!\n" +
     "#   Failed test 'it doesn't work!'\n" +
-    "#   at TestMoreTest.scala line 19.\n" +
+    "#   at TestMoreTest.scala line 21.\n" +
     "#          got: '1'\n" +
     "#     expected: '0'\n" +
     "ok 7\n" +
     "not ok 8\n" +
-    "#   Failed test at TestMoreTest.scala line 21.\n" +
+    "#   Failed test at TestMoreTest.scala line 23.\n" +
     "#          got: '1'\n" +
     "#     expected: '0'\n" +
     "# isnt\n" +
     "ok 9 - it works!\n" +
     "not ok 10 - it doesn't work!\n" +
     "#   Failed test 'it doesn't work!'\n" +
-    "#   at TestMoreTest.scala line 25.\n" +
+    "#   at TestMoreTest.scala line 27.\n" +
     "#          got: '1'\n" +
     "#     expected: anything else\n" +
     "ok 11\n" +
     "not ok 12\n" +
-    "#   Failed test at TestMoreTest.scala line 27.\n" +
+    "#   Failed test at TestMoreTest.scala line 29.\n" +
     "#          got: '1'\n" +
     "#     expected: anything else\n" +
     "# like\n" +
     "ok 13 - it works!\n" +
     "not ok 14 - it doesn't work!\n" +
     "#   Failed test 'it doesn't work!'\n" +
-    "#   at TestMoreTest.scala line 31.\n" +
+    "#   at TestMoreTest.scala line 33.\n" +
     "#                   'foo'\n" +
     "#     doesn't match 'bar'\n" +
     "ok 15\n" +
     "not ok 16\n" +
-    "#   Failed test at TestMoreTest.scala line 33.\n" +
+    "#   Failed test at TestMoreTest.scala line 35.\n" +
     "#                   'foo'\n" +
     "#     doesn't match 'bar'\n" +
     "    # unlike\n" +
     "    ok 1 - it works!\n" +
     "    not ok 2 - it doesn't work!\n" +
     "    #   Failed test 'it doesn't work!'\n" +
-    "    #   at TestMoreTest.scala line 38.\n" +
+    "    #   at TestMoreTest.scala line 40.\n" +
     "    #                   'foo'\n" +
     "    #           matches 'foo'\n" +
     "    ok 3\n" +
     "    not ok 4\n" +
-    "    #   Failed test at TestMoreTest.scala line 40.\n" +
+    "    #   Failed test at TestMoreTest.scala line 42.\n" +
     "    #                   'foo'\n" +
     "    #           matches 'foo'\n" +
     "    1..4\n" +
     "    # Looks like you failed 2 tests of 4.\n" +
     "not ok 17 - unlikes\n" +
     "#   Failed test 'unlikes'\n" +
-    "#   at TestMoreTest.scala line 35.\n" +
+    "#   at TestMoreTest.scala line 37.\n" +
     "# pass\n" +
     "ok 18 - it works!\n" +
     "ok 19\n" +
@@ -134,9 +136,9 @@ class TestMoreTest extends TestMore {
     "# fail\n" +
     "not ok 22 - it doesn't work # TODO not working yet\n" +
     "#   Failed (TODO) test 'it doesn't work'\n" +
-    "#   at TestMoreTest.scala line 54.\n" +
+    "#   at TestMoreTest.scala line 56.\n" +
     "not ok 23 # TODO not working yet\n" +
-    "#   Failed (TODO) test at TestMoreTest.scala line 55.\n" +
+    "#   Failed (TODO) test at TestMoreTest.scala line 57.\n" +
     "1..23\n" +
     "# Looks like you failed 9 tests of 23.\n"
 
diff --git a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala b/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala
index 8eab0d2..423b823 100644
--- a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala
+++ b/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala
@@ -102,34 +102,52 @@ class TestBuilderTest extends TestMore {
     val output = new ByteArrayOutputStream
     val oldOut = Console.out
     val oldErr = Console.err
-    Console.withOut(output) {
-      Console.withErr(output) {
-        val builder = new TestBuilder
-        Console.withOut(oldOut) {
-          Console.withErr(oldErr) {
-            ok(!builder.isPassing)
-          }
+
+    is(
+      Console.withOut(output) {
+        Console.withErr(output) {
+          val builder = new TestBuilder
+          builder.doneTesting
         }
-        builder.ok(true)
-        Console.withOut(oldOut) {
-          Console.withErr(oldErr) {
-            ok(builder.isPassing)
-          }
+      },
+      false
+    )
+
+    is(
+      Console.withOut(output) {
+        Console.withErr(output) {
+          val builder = new TestBuilder
+          builder.ok(true)
+          builder.doneTesting
         }
-        builder.ok(false)
-        Console.withOut(oldOut) {
-          Console.withErr(oldErr) {
-            ok(!builder.isPassing)
-          }
+      },
+      true
+    )
+
+    is(
+      Console.withOut(output) {
+        Console.withErr(output) {
+          val builder = new TestBuilder
+          builder.ok(true)
+          builder.ok(false)
+          builder.doneTesting
         }
-        builder.ok(true)
-        Console.withOut(oldOut) {
-          Console.withErr(oldErr) {
-            ok(!builder.isPassing)
-          }
+      },
+      false
+    )
+
+    is(
+      Console.withOut(output) {
+        Console.withErr(output) {
+          val builder = new TestBuilder
+          builder.ok(true)
+          builder.ok(false)
+          builder.ok(true)
+          builder.doneTesting
         }
-      }
-    }
+      },
+      false
+    )
   }
 
   subtest ("bail out") {