hack together a multi-test harness

needs a *lot* of work, but the basic idea is there

5 files changed, 146 insertions, 24 deletions

diff --git a/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala b/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala
index 9f128d6..b0548e4 100644
--- a/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala
+++ b/src/main/scala/org/perl8/test/harness/MultiTestReporter.scala
@@ -1,12 +1,5 @@
 package org.perl8.test.harness
 
-import org.perl8.test.Test
-import Utils._
-
-trait MultiTestReporter extends Reporter {
-  def run (tests: Array[Test]): Int =
-    tests.map(run).sum min 255
-
-  def run (testNames: Array[String]): Int =
-    run(testNames.map(newInstance[Test]))
+trait MultiTestReporter {
+  def run (testNames: Seq[String]): Int
 }
diff --git a/src/main/scala/org/perl8/test/harness/Reporter.scala b/src/main/scala/org/perl8/test/harness/Reporter.scala
index 3bfb874..536636f 100644
--- a/src/main/scala/org/perl8/test/harness/Reporter.scala
+++ b/src/main/scala/org/perl8/test/harness/Reporter.scala
@@ -1,13 +1,5 @@
 package org.perl8.test.harness
 
-import java.io.OutputStream
-
-import org.perl8.test.Test
-import Utils._
-
 trait Reporter {
-  def run (test: Test): Int
-
-  def run (testName: String): Int =
-    run(newInstance[Test](testName))
+  def run (testName: String): Int
 }
diff --git a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala
new file mode 100644
index 0000000..16a0f71
--- /dev/null
+++ b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala
@@ -0,0 +1,112 @@
+package org.perl8.test.harness
+
+import java.io.{OutputStream,ByteArrayOutputStream}
+
+import org.perl8.test.Test
+import org.perl8.test.tap
+import Utils._
+
+class SummaryReporter extends MultiTestReporter {
+  def run (testNames: Seq[String]): Int = {
+    val maxLength = testNames.map(_.length).max
+    val (exits, results) = testNames.map { name =>
+      print(name + ("." * (maxLength - name.length)) + "... ")
+      val out = new ByteArrayOutputStream
+      val test = newInstance[Test, OutputStream](name, out)
+      val exitCode = test.run
+      val result = tap.Consumer.parse(out)
+      if (exitCode == 0) {
+        println("ok")
+      }
+      else {
+        val results = result.results
+        val failed = results.filter(t => !t.passed && !t.directive.isDefined)
+        println("Dubious, test returned " + exitCode)
+        println("Failed " + failed.length + "/" + results.length + " subtests")
+      }
+      (exitCode, result)
+    }.unzip
+
+    val exitMap   = (testNames zip exits).toMap
+    val resultMap = (testNames zip results).toMap
+
+    val exitCode = if (exits.filter(_ != 0).nonEmpty) { 1 } else { 0 }
+
+    if (exitCode == 0) {
+      println("All tests successful.")
+    }
+
+    val tests = results.map(_.results.length).sum
+    println("Files=" + testNames.length + ", Tests=" + tests)
+
+    val todoSucceeded = resultMap.mapValues { r =>
+      r.results.filter { t =>
+        t.directive match {
+          case Some(tap.TodoDirective(_)) => t.passed
+          case _                      => false
+        }
+      }
+    }.filter(_._2.length > 0)
+
+    val testsFailed = resultMap.mapValues { r =>
+      r.results.filter { t =>
+        t.directive match {
+          case None => !t.passed
+          case _    => false
+        }
+      }
+    }.filter(_._2.length > 0)
+
+    if (todoSucceeded.nonEmpty || testsFailed.nonEmpty) {
+      println("")
+      println("Test Summary Report")
+      println("-------------------")
+      for (name <- (todoSucceeded ++ testsFailed).keys) {
+        val result = resultMap(name)
+        println(
+          name + (" " * (maxLength - name.length)) + " " +
+          "(Tests: " + result.results.length + " " +
+          "Failed: " + testsFailed.getOrElse(name, Seq()).length + ")"
+        )
+        if (testsFailed.isDefinedAt(name)) {
+          val fails = testsFailed(name)
+          println(
+            "  Failed test" + (if (fails.length > 1) "s" else "") + ": " +
+            fails.map(_.number).mkString(", ")
+          )
+        }
+        if (todoSucceeded.isDefinedAt(name)) {
+          println(
+            "  TODO passed: " +
+            todoSucceeded(name).map(_.number).mkString(", ")
+          )
+        }
+        if (exitMap(name) != 0) {
+          println("  Non-zero exit status: " + exitMap(name))
+        }
+      }
+    }
+
+    if (exitCode == 0) {
+      println("Result: PASS")
+    }
+    else {
+      println("Result: FAIL")
+      val failedFiles = results.filter { r =>
+        r.results.exists { t =>
+          t.directive match {
+            case None => !t.passed
+            case _    => false
+          }
+        }
+      }.length
+      val failedTests = testsFailed.mapValues(_.length).values.sum
+      println(
+        "Failed " + failedFiles + "/" + testNames.length + " test programs. " +
+        failedTests + "/" + tests + " subtests failed."
+      )
+    }
+
+    exitCode
+  }
+}
diff --git a/src/main/scala/org/perl8/test/harness/TAPReporter.scala b/src/main/scala/org/perl8/test/harness/TAPReporter.scala
index 94d0b8a..e478880 100644
--- a/src/main/scala/org/perl8/test/harness/TAPReporter.scala
+++ b/src/main/scala/org/perl8/test/harness/TAPReporter.scala
@@ -1,8 +1,9 @@
 package org.perl8.test.harness
 
 import org.perl8.test.Test
+import Utils._
 
 class TAPReporter extends Reporter {
-  def run (test: Test): Int =
-    test.run
+  def run (testName: String): Int =
+    newInstance[Test](testName).run
 }
diff --git a/src/main/scala/org/perl8/test/harness/TestHarness.scala b/src/main/scala/org/perl8/test/harness/TestHarness.scala
index 2ca33c6..2906150 100644
--- a/src/main/scala/org/perl8/test/harness/TestHarness.scala
+++ b/src/main/scala/org/perl8/test/harness/TestHarness.scala
@@ -5,10 +5,34 @@ import Utils._
 
 object TestHarness {
   def main (args: Array[String]) {
-    val reporterName = args(0)
-    val testName = args(1)
-    val reporter = newInstance[Reporter](reporterName)
-    val exitCode = reporter.run(testName)
+    val (reporterName, idx, multi) = if (args.length >= 2 && args(0) == "-r") {
+      (args(1), 2, false)
+    }
+    else if (args.length >= 2 && args(0) == "-R") {
+      (args(1), 2, true)
+    }
+    else if (args.length > 1) {
+      ("org.perl8.test.harness.SummaryReporter", 0, true)
+    }
+    else if (args.length == 1) {
+      ("org.perl8.test.harness.TAPReporter", 0, false)
+    }
+    else {
+      println("No tests specified!")
+      sys.exit(1)
+    }
+
+    val exitCode = if (multi) {
+      val testNames = args
+      val reporter = newInstance[MultiTestReporter](reporterName)
+      reporter.run(testNames)
+    }
+    else {
+      val testName = args(0)
+      val reporter = newInstance[Reporter](reporterName)
+      reporter.run(testName)
+    }
+
     sys.exit(exitCode)
   }
 }