diff options
Diffstat (limited to 'src/main/scala/com/iinteractive/test/tap')
7 files changed, 765 insertions, 0 deletions
diff --git a/src/main/scala/com/iinteractive/test/tap/Consumer.scala b/src/main/scala/com/iinteractive/test/tap/Consumer.scala new file mode 100644 index 0000000..09a2f0a --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Consumer.scala @@ -0,0 +1,131 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** Contains a method to parse an individual line of TAP. */ +object Consumer { + /** Parses a line of TAP. + * + * @return A [[com.iinteractive.test.tap.Consumer.Line Line]] object + * corresponding to the parsed line. + */ + def parseLine (line: String): Line = { + commentRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, text) => new CommentLine(text, indent) + } + }.getOrElse { + planRx.findFirstMatchIn(line).map { m => + m.subgroups match { + case Seq(indent, p, null) => + new PlanLine(NumericPlan(p.toInt), indent) + case Seq(indent, _, skip) => + new PlanLine(SkipAll(skip), indent) + } + }.getOrElse { + resultRx.findFirstMatchIn(line).map { m => + val indent = m.group(1) + val passed = m.group(2) == null + val number = m.group(3).toInt + val description = m.group(4) match { + case null => "" + case s => s.trim + } + val directive = (m.group(5), m.group(6)) match { + case (null, null) => None + case (d, r) => { + val reason = if (r == null) "" else r + """(?i:skip)""".r.findFirstIn(d) match { + case Some(_) => Some(new SkipDirective(Some(reason))) + case None => Some(new TodoDirective(Some(reason))) + } + } + } + val result = new TestResult( + passed, + number, + description, + directive, + None + ) + new ResultLine(result, indent) + }.getOrElse { + throw ParseException("Couldn't parse line: " + line) + } + } + } + } + + /** The representation of a parsed line of TAP. */ + sealed trait Line { + /** The meaningful portion of the TAP line. */ + def contents: String + /** The indentation of the TAP line (used in subtests). */ + def indent: String + + /** The line itself that was parsed. */ + override def toString: String = + indent + contents + } + + /** A parsed TAP line containing a comment. + * + * @param text The text of the comment (not including the `#`). + */ + case class CommentLine private[Consumer] ( + text: String, + indent: String + ) extends Line { + def contents = "# " + text + } + + /** A parsed TAP line containing a test plan. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] that this line + * represents. + */ + case class PlanLine private[Consumer] ( + plan: Plan, + indent: String + ) extends Line { + def contents = { + val count = plan.plan + val comment = plan match { + case SkipAll(m) => " # SKIP " + m + case _ => "" + } + indent + "1.." + count + comment + } + } + + /** A parsed TAP line containing a test result. + * + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * that this line represents. + */ + case class ResultLine private[Consumer] ( + result: TestResult, + indent: String + ) extends Line { + def contents = { + val success = (if (result.passed) "ok" else "not ok") + " " + val number = result.number + " " + val description = result.description match { + case "" => "" + case s => s + " " + } + val directive = result.directive.map { d => + d match { + case TodoDirective(m) => "# TODO " + m + case SkipDirective(m) => "# skip " + m + } + }.getOrElse("") + indent + success + number + description + directive + } + } + + private val commentRx = """^(\s*)#\s*(.*)""".r + private val planRx = """^(\s*)1..(\d+)\s*(?:# SKIP (.*))?""".r + private val resultRx = + """^(\s*)(not )?ok (\d+)\s*([^#]+)?(?:#\s*(?i:(skip|todo))\s+(.*))?""".r +} diff --git a/src/main/scala/com/iinteractive/test/tap/Parser.scala b/src/main/scala/com/iinteractive/test/tap/Parser.scala new file mode 100644 index 0000000..7bc44d3 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Parser.scala @@ -0,0 +1,232 @@ +package com.iinteractive.test.tap + +import java.io.{ByteArrayInputStream,InputStream,OutputStream} +import scala.annotation.tailrec +import scala.io.Source +import scala.util.parsing.combinator._ +import scala.util.parsing.input.{Position,Reader} + +import com.iinteractive.test.Plan +import com.iinteractive.test.tap.Consumer._ + +/** This class parses a TAP stream. It can either parse it all at once (from a + * string), or it can be used as a streaming parser, where TAP events are + * emitted through a given callback. + */ +class Parser private ( + cb: TAPEvent => Unit, + indent: String +) { + /** Creates a parser instance. + * @param cb The event handler callback. It will be called after each + * meaningful line of TAP, with a + * [[com.iinteractive.test.tap.TAPEvent TAPEvent]] instance + * representing the event that was just parsed. + */ + def this (cb: TAPEvent => Unit = e => ()) = + this(cb, "") + + private def this (indent: String) = + this(e => (), indent) + + /** Parses TAP from an input stream. This variant will actually parse lines + * as they are available to read from the input stream, so this can be used + * as a streaming parser. + */ + def parse (input: InputStream): TAPResult = { + import parser._ + + cb(StartEvent) + tap(new LineReader(input)) match { + case Success(result, _) => { + cb(EndEvent(result)) + result + } + case failure: NoSuccess => throw new ParseException(failure.msg) + } + } + + /** Parses TAP contained in a string. This isn't useful for incremental + * parsing, because the entire input string must be created before + * parsing can begin. + */ + def parse (input: String): TAPResult = + parse(new ByteArrayInputStream(input.getBytes)) + + /** Parses TAP from an output stream. + * + * @todo Doesn't currently work as a streaming parser, since it just + * collects the entire output as a string and feeds it to the parser + * for strings. This could likely be improved. + */ + def parse (input: OutputStream): TAPResult = + parse(input.toString) + + private val parser = new TAPParser(cb, indent) + + private class TAPParser ( + cb: TAPEvent => Unit, + indent: String + ) extends Parsers { + type Elem = Line + + def tap: Parser[TAPResult] = + planFirst | planLast + + private def planFirst: Parser[TAPResult] = + plan ~ rep(result) ^^ { case plan ~ results => + new TAPResult(plan, results) + } + + private def planLast: Parser[TAPResult] = + rep(result) ~ plan ^^ { case results ~ plan => + new TAPResult(plan, results) + } + + private def plan: Parser[Plan] = + planLine ^^ { p => + cb(PlanEvent(p.plan)) + p.plan + } + + private def result: Parser[TestResult] = + simpleResult | subtestResult + + private def simpleResult: Parser[TestResult] = + resultLine ^^ { r => + cb(ResultEvent(r.result)) + r.result + } + + private def subtestResult: Parser[TestResult] = + subtest ~ simpleResult ^^ { case subtest ~ simpleResult => + new TestResult( + simpleResult.passed, + simpleResult.number, + simpleResult.description, + simpleResult.directive, + Some(subtest) + ) + } + + private def subtest: Parser[TAPResult] = + LineParser("subtest") { in => + // can't just return the result directly, because it's of a different + // type (the path dependent type associated with the new Parser + // instance we create here, rather than the path dependent type + // associated with this) + val subParser = new TAPParser( + e => (), + in.first.indent + ) + subParser.tap(in) match { + case subParser.Success(p, rest) => Success(p, rest) + case subParser.Failure(m, rest) => Failure(m, rest) + case subParser.Error(m, rest) => Error(m, rest) + } + } + + private def planLine: Parser[PlanLine] = LineParser("plan") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: PlanLine => + Success(p, in.rest) + case _ => + Failure("Plan line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Plan line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def resultLine: Parser[ResultLine] = LineParser("result") { in => + val line = in.first + if (line.indent == indent) { + line match { + case p: ResultLine => + Success(p, in.rest) + case _ => + Failure("Result line expected, but '" + line + "' found", in) + } + } + else { + Failure( + "Result line expected, but " + + "'" + line + "' has incorrect indentation", + in + ) + } + } + + private def LineParser[T] (lineType: String)( + body: Input => ParseResult[T] + ): Parser[T] = Parser { in => + if (in.atEnd) { + Failure(lineType + " line expected, but end of input found", in) + } + else { + body(in) + } + } + } + + private class LineReader ( + in: Iterator[Char], + lineNum: Int + ) extends Reader[Line] { + def this (in: InputStream) = + this(Source.fromInputStream(in), 1) + + def atEnd: Boolean = + nextLine.isEmpty + + def first: Line = + nextLine.getOrElse(throw new RuntimeException("read from empty input")) + + lazy val pos = + new LinePosition(lineNum, nextLine.map(_.toString).getOrElse("")) + + lazy val rest: Reader[Line] = + new LineReader(remainingStream, lineNum + 1) + + private def nextLine: Option[Line] = + state._1 + + private def remainingStream: Iterator[Char] = + state._2 + + private lazy val state: (Option[Line], Iterator[Char]) = + readNextLine(in) + + @tailrec + private def readNextLine ( + stream: Iterator[Char] + ): (Option[Line], Iterator[Char]) = { + if (stream.hasNext) { + val (line, rest) = stream.span(_ != '\n') match { + case (l, r) => (parseLine(l.mkString), r.drop(1)) + } + line match { + case _: CommentLine => readNextLine(rest) + case other => (Some(other), rest) + } + } + else { + (None, in) + } + } + } + + private case class LinePosition ( + line: Int, + lineContents: String + ) extends Position { + def column: Int = 1 + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/Producer.scala b/src/main/scala/com/iinteractive/test/tap/Producer.scala new file mode 100644 index 0000000..3ed8d7e --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/Producer.scala @@ -0,0 +1,63 @@ +package com.iinteractive.test.tap + +/** Contains functions for producing individual lines of TAP. */ +object Producer { + import com.iinteractive.test.Plan + + /** Returns a test result. + * + * @example `ok 4` + */ + def result (cond: Boolean, num: Int): String = + (if (cond) "ok " else "not ok ") + num + + /** Returns a test result that contains a description. + * + * @example `ok 28 - our test succeeded` + */ + def result (cond: Boolean, num: Int, desc: String): String = + result(cond, num) + " " + desc + + /** Returns a todo test result. + * + * @example `not ok 1 # TODO this doesn't work yet` + */ + def todoResult (cond: Boolean, num: Int, todo: String): String = + result(cond, num) + " # TODO " + todo + + /** Returns a todo test result that contains a description. + * + * @example `not ok 18 - test the feature # TODO can't figure this out` + */ + def todoResult (cond: Boolean, num: Int, desc: String, todo: String): String = + result(cond, num, desc) + " # TODO " + todo + + /** Returns a skipped test result with a reason. + * + * @example `ok 4 # skip this test won't run here` + */ + def skip (num: Int, reason: String): String = + "ok " + num + " # skip " + reason + + /** Returns a comment. + * + * @example `# this is a comment` + */ + def comment (message: String): String = + message.split("\n").map(m => "# " + m).mkString("\n") + + /** Returns a test plan. + * + * @example `1..5` ([[com.iinteractive.test.NumericPlan NumericPlan]]) + * @example `1..0 # SKIP don't run this test` ([[com.iinteractive.test.SkipAll SkipAll]]) + */ + def plan (plan: Plan): String = + plan.skipAll.map(m => "1..0 # SKIP " + m).getOrElse("1.." + plan.plan) + + /** Returns a bail out with a reason. + * + * @example `Bail out! Not supported on this platform.` + */ + def bailOut (message: String): String = + "Bail out! " + message +} diff --git a/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala new file mode 100644 index 0000000..1c2e88d --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPEvent.scala @@ -0,0 +1,40 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.Plan + +/** An event emitted while parsing a TAP stream. */ +sealed trait TAPEvent + +/** The start of a TAP stream. */ +case object StartEvent extends TAPEvent + +/** The end of a TAP stream. + * @param result The [[com.iinteractive.test.tap.TAPResult TAPResult]] + * containing information about all of the tests which just + * finished running. This will be the same thing that is + * returned by the call to + * [[com.iinteractive.test.tap.Parser Parser]]'s `parse` + * method. + */ +case class EndEvent private[tap] (result: TAPResult) extends TAPEvent + +/** An individual test result. + * @param result The [[com.iinteractive.test.tap.TestResult TestResult]] + * containing information about the corresponding test. + */ +case class ResultEvent private[tap] (result: TestResult) extends TAPEvent + +/** A test plan. + * @param plan The [[com.iinteractive.test.Plan Plan]] corresponding to the + * line that was parsed. + */ +case class PlanEvent private[tap] (plan: Plan) extends TAPEvent + +/** The start of a subtest (currently unused). */ +case object SubtestStartEvent extends TAPEvent + +/** The end of a subtest (currently unused). */ +case class SubtestEndEvent private[tap] (result: TestResult) extends TAPEvent + +/** A comment (currently unused). */ +case class CommentEvent private[tap] (text: String) extends TAPEvent diff --git a/src/main/scala/com/iinteractive/test/tap/TAPResult.scala b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala new file mode 100644 index 0000000..c3c4926 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TAPResult.scala @@ -0,0 +1,94 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test.{Plan,NumericPlan,SkipAll} + +/** The summarized results of a TAP stream. Contains the + * [[com.iinteractive.test.Plan Plan]] that was given, as well as a list of + * [[com.iinteractive.test.tap.TestResult TestResults]] corresponding to each + * of the tests in the stream. + * + * @param plan The [[com.iinteractive.test.Plan Plan]] from the TAP stream + * @param results The list of + * [[com.iinteractive.test.tap.TestResult TestResults]] from + * the TAP stream + */ +class TAPResult (val plan: Plan, val results: Seq[TestResult]) { + /** Returns true if the number of tests executed was compatible with the + * provided test plan. + */ + val matchesPlan = plan match { + case NumericPlan(n) => results.length == n + case _ => results.length == 0 + } + + /** Returns the number of test failures in the TAP stream. */ + val fails = results.count { r => + !r.passed && !r.directive.isDefined + } + + /** Returns true if all of the tests passed. */ + val testsPassed = fails == 0 + + /** Returns true if the TAP stream overall passed. + * + * Differs from `testsPassed` in that it also takes into account things + * like invalid plans. + */ + val success = plan match { + case SkipAll(_) => true + case _ => results.length > 0 && fails == 0 && matchesPlan + } + + /** Returns the exit code to use if running this test on its own. + * + * Success is indicated by 0, invalid TAP streams (such as incorrect plans) + * by 255, and other kinds of failures by the failure count. + */ + val exitCode = + if (success) { + 0 + } + else if (!matchesPlan || results.length == 0) { + 255 + } + else { + fails + } +} + +/** The result of a single test. + * + * @param passed True if the test passed + * @param number The test number in the TAP stream + * @param description The test description + * @param directive The [[com.iinteractive.test.tap.Directive Directive]] + * (either skip or todo) that was provided for this test, + * if any + * @param subtest The [[com.iinteractive.test.tap.TAPResult]] for the + * subtest that this test corresponds to, if any + */ +class TestResult ( + val passed: Boolean, + val number: Int, + val description: String, + val directive: Option[Directive], + val subtest: Option[TAPResult] +) + +/** A modifier associated with a test result. This is indicated by a `#` at + * the end of the result line, followed by the type of directive, and an + * optional message. + */ +sealed trait Directive { + val message: Option[String] +} + +/** A directive indicating that this test was skipped. */ +case class SkipDirective private[tap] ( + message: Option[String] +) extends Directive + +/** A directive indicating that this test is known to fail. */ +case class TodoDirective private[tap] ( + message: Option[String] +) extends Directive diff --git a/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala new file mode 100644 index 0000000..62fdf02 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/TestBuilder.scala @@ -0,0 +1,196 @@ +package com.iinteractive.test.tap + +import com.iinteractive.test._ + +/** This class provides a convenient yet low level API for generating TAP + * streams. Each instance of this class handles a single TAP stream, and + * keeps track of things like the current test number for you. All + * TAP-producing methods write the TAP lines to `Console.out` or + * `Console.err`, so you can override those (via `Console.withOut` or + * `Console.withErr`). + */ +class TestBuilder private ( + plan: Plan, + indent: String, + terminalInUse: Boolean +) { + plan match { + case NoPlan => () + case p => outLine(Producer.plan(p)) + } + + /** Creates a new builder instance, and emits the corresponding plan line, + * unless the plan is not given. + * + * @param plan [[com.iinteractive.test.Plan plan]] for this test. + * @param terminalInUse Whether this test is being run from a harness which + * will not just be writing directly to the output. + * This will make things written to `Console.err` have + * a newline prepended, so that they always start on + * an empty line. + */ + def this (plan: Plan = NoPlan, terminalInUse: Boolean = false) = + this(plan, "", terminalInUse) + + /** Create a new TestBuilder instance, to be used to run a subtest. This new + * instance will have all of its lines prefixed by an additional level of + * indentation. This instance will still need to have `doneTesting` + * called on it, and the result of the subtest will still need to be + * reported as a separate test result through `ok`. + */ + def cloneForSubtest (newPlan: Plan): TestBuilder = + new TestBuilder(newPlan, indent + " ", terminalInUse) + + /** Reports a single test result to `Console.out`. */ + def ok (test: Boolean) { + state.ok(test) + outLine(Producer.result(test, state.currentTest)) + } + + /** Reports a single test result with description to `Console.out`. */ + def ok (test: Boolean, description: String) { + state.ok(test) + outLine(Producer.result(test, state.currentTest, description)) + } + + /** Reports a single TODO test result to `Console.out`. */ + def todo (todo: String, test: Boolean) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, todo)) + } + + /** Reports a single TODO test result with description to `Console.out`. */ + def todo (todo: String, test: Boolean, description: String) { + state.ok(true) + outLine(Producer.todoResult(test, state.currentTest, description, todo)) + } + + /** Reports a single skipped test result to `Console.out`. */ + def skip (reason: String) { + state.ok(true) + outLine(Producer.skip(state.currentTest, reason)) + } + + /** Writes a comment line to `Console.err`. This will allow it to be + * visible in most summarizing harnesses (which consume and parse + * everything that goes to `Console.out`). + */ + def diag (message: String) { + errLine(Producer.comment(message)) + } + + /** Write a comment line to `Console.out`. This will typically only be + * visible in the raw TAP stream. + */ + def note (message: String) { + outLine(Producer.comment(message)) + } + + /** Abort the current test, with a message. */ + def bailOut (message: String) { + val bailOutMessage = Producer.bailOut(message) + outLine(bailOutMessage) + throw new BailOutException(bailOutMessage) + } + + /** Finalize the current builder instance. This writes the auto-calculated + * plan to `Console.out` if the plan type was `NoPlan` and reports a + * summary of the test results as a comment to `Console.err`. + * + * @return whether or not the test class as a whole passed. + */ + def doneTesting: Boolean = { + plan match { + case NumericPlan(_) => printErrors + case SkipAll(_) => () + case NoPlan => { + outLine(Producer.plan(state.currentTest)) + printErrors + } + } + state.isPassing + } + + /** The exit code to use, in harnesses that run a single test. Passing tests + * return 0, invalid tests (such as running a different number of tests + * than planned) return 255, and all others return the number of failed + * tests. + */ + def exitCode: Int = + if (state.isPassing) { + 0 + } + else if (!state.matchesPlan || state.currentTest == 0) { + 255 + } + else { + state.failCount + } + + private def printErrors { + if (!state.matchesPlan) { + val planCount = (plan match { + case NoPlan => state.currentTest + case p => p.plan + }) + val planned = planCount + " test" + (if (planCount > 1) "s" else "") + val ran = state.currentTest + diag("Looks like you planned " + planned + " but ran " + ran + ".") + } + + if (state.currentTest == 0) { + diag("No tests run!") + } + + if (state.failCount > 0) { + val count = state.failCount + val fails = count + " test" + (if (count > 1) "s" else "") + val total = + state.currentTest + (if (state.matchesPlan) "" else " run") + diag("Looks like you failed " + fails + " of " + total + ".") + } + } + + private val state = new TestState + + private def outLine (str: String) { + Console.out.println(withIndent(str)) + } + + private def errLine (str: String) { + if (terminalInUse) { + Console.err.print("\n") + } + Console.err.println(withIndent(str)) + } + + private def withIndent (str: String): String = + str.split("\n").map(s => indent + s).mkString("\n") + + private class TestState { + var passCount = 0 + var failCount = 0 + + def ok (cond: Boolean) { + if (cond) { + passCount += 1 + } + else { + failCount += 1 + } + } + + def currentTest: Int = + failCount + passCount + + def matchesPlan: Boolean = plan match { + case NumericPlan(p) => p.plan == currentTest + case _ => true + } + + def isPassing: Boolean = plan match { + case SkipAll(_) => true + case _ => currentTest > 0 && failCount == 0 && matchesPlan + } + } +} diff --git a/src/main/scala/com/iinteractive/test/tap/package.scala b/src/main/scala/com/iinteractive/test/tap/package.scala new file mode 100644 index 0000000..312f996 --- /dev/null +++ b/src/main/scala/com/iinteractive/test/tap/package.scala @@ -0,0 +1,9 @@ +package com.iinteractive.test + +/** Classes for TAP generation and parsing. */ +package object tap { + /** Exception representing an error during parsing. It is thrown when a TAP + * line isn't recognized. + */ + case class ParseException (message: String) extends RuntimeException(message) +} |