From 073d5a06c07c85c0d1794c4a15d564e8463b31fc Mon Sep 17 00:00:00 2001 From: Jesse Luehrs Date: Sat, 23 Feb 2013 16:09:06 -0600 Subject: tests shouldn't produce anything other than tap the harness can parse the tap to figure out whatever it needs from that --- src/main/scala/org/perl8/test/Test.scala | 2 +- src/main/scala/org/perl8/test/TestMore.scala | 12 +--- src/main/scala/org/perl8/test/Utils.scala | 7 +-- .../org/perl8/test/harness/SummaryReporter.scala | 34 ++++++----- .../scala/org/perl8/test/harness/TAPReporter.scala | 20 ++++++- .../scala/org/perl8/test/sbt/SBTReporter.scala | 10 ++-- src/main/scala/org/perl8/test/tap/Consumer.scala | 27 ++++++++- .../scala/org/perl8/test/tap/TestBuilder.scala | 7 +-- src/test/scala/org/perl8/test/ExtensionTest.scala | 10 ++-- src/test/scala/org/perl8/test/TestMoreTest.scala | 32 ++++++----- .../scala/org/perl8/test/tap/TestBuilderTest.scala | 66 ++++++++++++++-------- 11 files changed, 137 insertions(+), 90 deletions(-) (limited to 'src') diff --git a/src/main/scala/org/perl8/test/Test.scala b/src/main/scala/org/perl8/test/Test.scala index 93933dd..cc79df8 100644 --- a/src/main/scala/org/perl8/test/Test.scala +++ b/src/main/scala/org/perl8/test/Test.scala @@ -6,5 +6,5 @@ trait Test { /** Runs the test. The TAP stream will be written to Console.out and * Console.err, so you can swap these out as required in order to parse it. */ - def run (): Int + def run (): Unit } diff --git a/src/main/scala/org/perl8/test/TestMore.scala b/src/main/scala/org/perl8/test/TestMore.scala index 487b023..2253d31 100644 --- a/src/main/scala/org/perl8/test/TestMore.scala +++ b/src/main/scala/org/perl8/test/TestMore.scala @@ -16,22 +16,13 @@ class TestMore (plan: Option[Plan] = None) extends Test with DelayedInit { testBody = () => body } - def run (): Int = { + def run () { if (testBody == null) { delayedInit { } } testBody() builder.doneTesting - if (builder.isPassing) { - 0 - } - else if (builder.failedTests == 0) { - 255 - } - else { - builder.failedTests - } } def ok (cond: Boolean, desc: Message = NoMessage): Boolean = { @@ -125,7 +116,6 @@ class TestMore (plan: Option[Plan] = None) extends Test with DelayedInit { ) body builder.doneTesting - builder.isPassing } finally { builder = oldBuilder diff --git a/src/main/scala/org/perl8/test/Utils.scala b/src/main/scala/org/perl8/test/Utils.scala index 5135688..31a66b1 100644 --- a/src/main/scala/org/perl8/test/Utils.scala +++ b/src/main/scala/org/perl8/test/Utils.scala @@ -10,13 +10,10 @@ object Utils { } case class NumericPlan ( - override val plan: Int, - override val message: Option[String] = None + override val plan: Int ) extends Plan { override val skipAll = false - - def this (plan: Int, message: String) = - this(plan, Some(message)) + override val message = None } case class SkipAll ( diff --git a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala index 23e5244..460059a 100644 --- a/src/main/scala/org/perl8/test/harness/SummaryReporter.scala +++ b/src/main/scala/org/perl8/test/harness/SummaryReporter.scala @@ -10,12 +10,12 @@ import Utils._ class SummaryReporter extends MultiTestReporter { def run (testNames: Seq[String]): Int = { val results = runTests(testNames) - val success = results.values.map(_._1).forall(_ == 0) + val success = results.values.forall(_.success) printTestSummary(success, results) if (success) 0 else 1 } - def runTests (testNames: Seq[String]): Map[String, (Int, TAPResult)] = { + def runTests (testNames: Seq[String]): Map[String, TAPResult] = { val maxLength = testNames.map(_.length).max testNames.map { name => @@ -23,12 +23,12 @@ class SummaryReporter extends MultiTestReporter { val out = new ByteArrayOutputStream val test = newInstance[Test](name) - val exitCode = Console.withOut(out) { + Console.withOut(out) { test.run } val result = tap.Consumer.parse(out) - if (exitCode == 0) { + if (result.success) { println("ok") } else { @@ -37,17 +37,17 @@ class SummaryReporter extends MultiTestReporter { !t.passed && !t.directive.isDefined } - println("Dubious, test returned " + exitCode) + println("Dubious, test returned " + result.exitCode) println("Failed " + failed + "/" + results + " subtests") } - name -> (exitCode, result) + name -> result }.toMap } def printTestSummary ( success: Boolean, - results: Map[String, (Int, TAPResult)] + results: Map[String, TAPResult] ) { printSuccess(success) printShortSummary(results) @@ -61,16 +61,14 @@ class SummaryReporter extends MultiTestReporter { } } - private def printShortSummary (results: Map[String, (Int, TAPResult)]) { + private def printShortSummary (results: Map[String, TAPResult]) { val files = results.size - val tests = results.values.map(_._2.results.length).sum + val tests = results.values.map(_.results.length).sum println("Files=" + files + ", Tests=" + tests) } - private def printLongSummary (results: Map[String, (Int, TAPResult)]) { - val resultMap = results.map { case (s, r) => s -> r._2 } - - val todoSucceeded = resultMap.mapValues { r => + private def printLongSummary (results: Map[String, TAPResult]) { + val todoSucceeded = results.mapValues { r => r.results.filter { t => t.directive match { case Some(TodoDirective(_)) => t.passed @@ -79,7 +77,7 @@ class SummaryReporter extends MultiTestReporter { } }.filter(_._2.length > 0) - val testsFailed = resultMap.mapValues { r => + val testsFailed = results.mapValues { r => r.results.filter { t => t.directive match { case None => !t.passed @@ -98,7 +96,7 @@ class SummaryReporter extends MultiTestReporter { val maxLength = testNames.map(_.length).max for (name <- testNames) { - val result = resultMap(name) + val result = results(name) println( name + (" " * (maxLength - name.length)) + " " + @@ -122,7 +120,7 @@ class SummaryReporter extends MultiTestReporter { ) } - val exitCode = results(name)._1 + val exitCode = results(name).exitCode if (exitCode != 0) { println(" Non-zero exit status: " + exitCode) } @@ -132,7 +130,7 @@ class SummaryReporter extends MultiTestReporter { private def printPassFail ( success: Boolean, - results: Map[String, (Int, TAPResult)] + results: Map[String, TAPResult] ) { if (success) { println("Result: PASS") @@ -140,7 +138,7 @@ class SummaryReporter extends MultiTestReporter { else { println("Result: FAIL") - val testResults = results.values.map(_._2) + val testResults = results.values val testsFailed = testResults.map { r => r.results.count { t => diff --git a/src/main/scala/org/perl8/test/harness/TAPReporter.scala b/src/main/scala/org/perl8/test/harness/TAPReporter.scala index e478880..574dd39 100644 --- a/src/main/scala/org/perl8/test/harness/TAPReporter.scala +++ b/src/main/scala/org/perl8/test/harness/TAPReporter.scala @@ -1,9 +1,25 @@ package org.perl8.test.harness +import java.io.ByteArrayOutputStream + +import org.perl8.test.tap import org.perl8.test.Test import Utils._ class TAPReporter extends Reporter { - def run (testName: String): Int = - newInstance[Test](testName).run + def run (testName: String): Int = { + val out = new ByteArrayOutputStream + Console.withOut(out) { + Console.withErr(out) { + newInstance[Test](testName).run + } + } + + // XXX this is wrong: it sends everything to stdout + // need to write a tee-like outputstream to fix it + print(out) + + val result = tap.Consumer.parse(out) + result.exitCode + } } diff --git a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala b/src/main/scala/org/perl8/test/sbt/SBTReporter.scala index 6307f55..2f47af2 100644 --- a/src/main/scala/org/perl8/test/sbt/SBTReporter.scala +++ b/src/main/scala/org/perl8/test/sbt/SBTReporter.scala @@ -41,14 +41,14 @@ class SBTReporter ( eventHandler.handle(event) } - if (result.results.exists { r => !r.passed && !r.directive.isDefined }) { - logError(testName + " failed.") - 1 - } - else { + if (result.success) { logInfo(testName + " succeeded.") 0 } + else { + logError(testName + " failed.") + 1 + } } private def logDebug (msg: String) { diff --git a/src/main/scala/org/perl8/test/tap/Consumer.scala b/src/main/scala/org/perl8/test/tap/Consumer.scala index 7a0f7aa..2c542f1 100644 --- a/src/main/scala/org/perl8/test/tap/Consumer.scala +++ b/src/main/scala/org/perl8/test/tap/Consumer.scala @@ -135,7 +135,32 @@ case class TestResult ( val subtest: Option[TAPResult] ) -class TAPResult (val plan: Plan, val results: Seq[TestResult]) +class TAPResult (val plan: Plan, val results: Seq[TestResult]) { + val correctPlan = plan match { + case NumericPlan(n) => results.length == n + case SkipAll(_) => results.length == 0 + } + + val fails = results.count { r => + !r.passed && !r.directive.isDefined + } + + val testsPassed = fails == 0 + + val success = + correctPlan && testsPassed + + val exitCode = + if (success) { + 0 + } + else if (!correctPlan) { + 255 + } + else { + fails + } +} case class ParseException ( val message: String diff --git a/src/main/scala/org/perl8/test/tap/TestBuilder.scala b/src/main/scala/org/perl8/test/tap/TestBuilder.scala index 203080c..b1d8579 100644 --- a/src/main/scala/org/perl8/test/tap/TestBuilder.scala +++ b/src/main/scala/org/perl8/test/tap/TestBuilder.scala @@ -51,13 +51,13 @@ class TestBuilder ( throw new BailOutException(message.getOrElse("")) } - def doneTesting () { + def doneTesting (): Boolean = { plan match { case None => outLine(Producer.plan(state.currentTest - 1)) case _ => () } - if (!isPassing) { + if (!state.isPassing) { if (!state.matchesPlan) { val planCount = (plan match { case Some(p) => p.plan @@ -80,10 +80,9 @@ class TestBuilder ( diag("Looks like you failed " + fails + " of " + total + ".") } } - } - def isPassing: Boolean = state.isPassing + } def failedTests: Int = state.failCount diff --git a/src/test/scala/org/perl8/test/ExtensionTest.scala b/src/test/scala/org/perl8/test/ExtensionTest.scala index e616d3e..6455e16 100644 --- a/src/test/scala/org/perl8/test/ExtensionTest.scala +++ b/src/test/scala/org/perl8/test/ExtensionTest.scala @@ -2,6 +2,8 @@ package org.perl8.test import java.io.ByteArrayOutputStream +import org.perl8.test.tap.Consumer + trait NumberZero { this: TestMore => def is_zero (i: Int, desc: String): Boolean = { withLevel(1) { @@ -27,25 +29,25 @@ class ExtensionTest extends TestMore { } val out = new ByteArrayOutputStream - val exitCode = Console.withOut(out) { + Console.withOut(out) { Console.withErr(out) { (new ExtensionTestTest).run } } - is(exitCode, 2) + is(Consumer.parse(out).exitCode, 2) val tap = "ok 1 - it's zero\n" + "not ok 2 - it's not zero\n" + "# Failed test 'it's not zero'\n" + - "# at ExtensionTest.scala line 24.\n" + + "# at ExtensionTest.scala line 26.\n" + "# got: '1'\n" + "# expected: '0'\n" + "ok 3 - the number is zero\n" + "not ok 4 - the number is zero\n" + "# Failed test 'the number is zero'\n" + - "# at ExtensionTest.scala line 26.\n" + + "# at ExtensionTest.scala line 28.\n" + "# got: '1'\n" + "# expected: '0'\n" + "1..4\n" + diff --git a/src/test/scala/org/perl8/test/TestMoreTest.scala b/src/test/scala/org/perl8/test/TestMoreTest.scala index 61bacad..a72c18b 100644 --- a/src/test/scala/org/perl8/test/TestMoreTest.scala +++ b/src/test/scala/org/perl8/test/TestMoreTest.scala @@ -2,6 +2,8 @@ package org.perl8.test import java.io.ByteArrayOutputStream +import org.perl8.test.tap.Consumer + class TestMoreTest extends TestMore { private object OutputContainer { val output = new ByteArrayOutputStream @@ -56,76 +58,76 @@ class TestMoreTest extends TestMore { } } - val exitCode = Console.withOut(OutputContainer.output) { + Console.withOut(OutputContainer.output) { Console.withErr(OutputContainer.output) { (new MyBasicTest).run } } - is(exitCode, 9, "got the right plan") + is(Consumer.parse(OutputContainer.output).exitCode, 9, "got the right plan") val expected = "# ok\n" + "ok 1 - it works!\n" + "not ok 2 - it doesn't work!\n" + "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line 13.\n" + + "# at TestMoreTest.scala line 15.\n" + "ok 3\n" + "not ok 4\n" + - "# Failed test at TestMoreTest.scala line 15.\n" + + "# Failed test at TestMoreTest.scala line 17.\n" + "# is\n" + "ok 5 - it works!\n" + "not ok 6 - it doesn't work!\n" + "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line 19.\n" + + "# at TestMoreTest.scala line 21.\n" + "# got: '1'\n" + "# expected: '0'\n" + "ok 7\n" + "not ok 8\n" + - "# Failed test at TestMoreTest.scala line 21.\n" + + "# Failed test at TestMoreTest.scala line 23.\n" + "# got: '1'\n" + "# expected: '0'\n" + "# isnt\n" + "ok 9 - it works!\n" + "not ok 10 - it doesn't work!\n" + "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line 25.\n" + + "# at TestMoreTest.scala line 27.\n" + "# got: '1'\n" + "# expected: anything else\n" + "ok 11\n" + "not ok 12\n" + - "# Failed test at TestMoreTest.scala line 27.\n" + + "# Failed test at TestMoreTest.scala line 29.\n" + "# got: '1'\n" + "# expected: anything else\n" + "# like\n" + "ok 13 - it works!\n" + "not ok 14 - it doesn't work!\n" + "# Failed test 'it doesn't work!'\n" + - "# at TestMoreTest.scala line 31.\n" + + "# at TestMoreTest.scala line 33.\n" + "# 'foo'\n" + "# doesn't match 'bar'\n" + "ok 15\n" + "not ok 16\n" + - "# Failed test at TestMoreTest.scala line 33.\n" + + "# Failed test at TestMoreTest.scala line 35.\n" + "# 'foo'\n" + "# doesn't match 'bar'\n" + " # unlike\n" + " ok 1 - it works!\n" + " not ok 2 - it doesn't work!\n" + " # Failed test 'it doesn't work!'\n" + - " # at TestMoreTest.scala line 38.\n" + + " # at TestMoreTest.scala line 40.\n" + " # 'foo'\n" + " # matches 'foo'\n" + " ok 3\n" + " not ok 4\n" + - " # Failed test at TestMoreTest.scala line 40.\n" + + " # Failed test at TestMoreTest.scala line 42.\n" + " # 'foo'\n" + " # matches 'foo'\n" + " 1..4\n" + " # Looks like you failed 2 tests of 4.\n" + "not ok 17 - unlikes\n" + "# Failed test 'unlikes'\n" + - "# at TestMoreTest.scala line 35.\n" + + "# at TestMoreTest.scala line 37.\n" + "# pass\n" + "ok 18 - it works!\n" + "ok 19\n" + @@ -134,9 +136,9 @@ class TestMoreTest extends TestMore { "# fail\n" + "not ok 22 - it doesn't work # TODO not working yet\n" + "# Failed (TODO) test 'it doesn't work'\n" + - "# at TestMoreTest.scala line 54.\n" + + "# at TestMoreTest.scala line 56.\n" + "not ok 23 # TODO not working yet\n" + - "# Failed (TODO) test at TestMoreTest.scala line 55.\n" + + "# Failed (TODO) test at TestMoreTest.scala line 57.\n" + "1..23\n" + "# Looks like you failed 9 tests of 23.\n" diff --git a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala b/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala index 8eab0d2..423b823 100644 --- a/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala +++ b/src/test/scala/org/perl8/test/tap/TestBuilderTest.scala @@ -102,34 +102,52 @@ class TestBuilderTest extends TestMore { val output = new ByteArrayOutputStream val oldOut = Console.out val oldErr = Console.err - Console.withOut(output) { - Console.withErr(output) { - val builder = new TestBuilder - Console.withOut(oldOut) { - Console.withErr(oldErr) { - ok(!builder.isPassing) - } + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.doneTesting } - builder.ok(true) - Console.withOut(oldOut) { - Console.withErr(oldErr) { - ok(builder.isPassing) - } + }, + false + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.doneTesting } - builder.ok(false) - Console.withOut(oldOut) { - Console.withErr(oldErr) { - ok(!builder.isPassing) - } + }, + true + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.ok(false) + builder.doneTesting } - builder.ok(true) - Console.withOut(oldOut) { - Console.withErr(oldErr) { - ok(!builder.isPassing) - } + }, + false + ) + + is( + Console.withOut(output) { + Console.withErr(output) { + val builder = new TestBuilder + builder.ok(true) + builder.ok(false) + builder.ok(true) + builder.doneTesting } - } - } + }, + false + ) } subtest ("bail out") { -- cgit v1.2.3-54-g00ecf