diff --git a/build.sc b/build.sc index 6828e665..d2899228 100644 --- a/build.sc +++ b/build.sc @@ -13,7 +13,7 @@ import de.tobiasroeser.mill.vcs.version.VcsVersion import $ivy.`com.github.lolgab::mill-mima::0.0.13` import com.github.lolgab.mill.mima._ -val scala31 = "3.1.3" +val scala31 = "3.2.2" val scala213 = "2.13.10" val scala212 = "2.12.17" val scala211 = "2.11.12" @@ -237,16 +237,17 @@ object perftests extends Module{ object bench2 extends PerfTestModule { def scalaVersion0 = scala213 def moduleDeps = Seq( - scalaparse.jvm(scala212).test, - pythonparse.jvm(scala212).test, - cssparse.jvm(scala212).test, - fastparse.jvm(scala212).test, + scalaparse.jvm(scala213).test, + pythonparse.jvm(scala213).test, + cssparse.jvm(scala213).test, + fastparse.jvm(scala213).test, ) } object benchScala3 extends PerfTestModule { def scalaVersion0 = scala31 + def sources = T.sources{ bench2.sources() } def moduleDeps = Seq( scalaparse.jvm(scala31).test, pythonparse.jvm(scala31).test, diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 9927e9fb..447e6c1c 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -54,7 +54,7 @@ object MacroImpls { if (ctx0.verboseFailures) { ctx0.aggregateMsg( startIndex, - Msgs(List(new Lazy(() => name.splice.value))), + Msgs(new Lazy(() => name.splice.value) :: Nil), ctx0.failureGroupAggregate, startIndex < ctx0.traceIndex ) diff --git a/fastparse/src/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala similarity index 73% rename from fastparse/src/fastparse/internal/RepImpls.scala rename to fastparse/src-2/fastparse/internal/RepImpls.scala index 26b1eecc..acdd2f5b 100644 --- a/fastparse/src/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -2,9 +2,10 @@ package fastparse.internal import fastparse.{Implicits, NoWhitespace, ParsingRun} - +import Util.{aggregateMsgInRep, aggregateMsgPostSep} import scala.annotation.tailrec + class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ def repX[V](min: Int = 0, sep: => ParsingRun[_] = null, @@ -148,7 +149,9 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ outerCut: Boolean, sepMsg: Msgs, lastAgg: Msgs): ParsingRun[V] = { + ctx.cut = precut | (count < min && outerCut) + if (count == 0 && actualMax == 0) ctx.freshSuccess(repeater.result(acc), startIndex) else { parse0() @@ -171,36 +174,34 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ if (verboseFailures) ctx.setMsg(startIndex, () => parsedMsg.render + ".rep" + (if(min == 0) "" else s"($min)")) res } + else if (!consumeWhitespace(whitespace, ctx, false)) ctx.asInstanceOf[ParsingRun[Nothing]] else { - if (whitespace ne NoWhitespace.noWhitespaceImplicit) Util.consumeWhitespace(whitespace, ctx) - - if (!ctx.isSuccess && ctx.cut) ctx.asInstanceOf[ParsingRun[Nothing]] - else { - ctx.cut = false - val sep1 = sep - val sepCut = ctx.cut - val endCut = outerCut | postCut | sepCut - if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) - else if (ctx.isSuccess) { - if (whitespace ne NoWhitespace.noWhitespaceImplicit) Util.consumeWhitespace(whitespace, ctx) - if (!ctx.isSuccess && sepCut) ctx.asInstanceOf[ParsingRun[Nothing]] - else rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) - } + ctx.cut = false + val sep1 = sep + val sepCut = ctx.cut + val endCut = outerCut | postCut | sepCut + if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) + else if (ctx.isSuccess) { + if (!consumeWhitespace(whitespace, ctx, sepCut)) ctx.asInstanceOf[ParsingRun[Nothing]] else { - val res = - if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) - else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - - if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) - res + rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) } } + else { + val res = + if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) + else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) + + if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) + res + } } } } } rec(ctx.index, 0, false, ctx.cut, null, null) } + def rep[V](min: Int, sep: => ParsingRun[_]) (implicit repeater: Implicits.Repeater[T, V], @@ -236,9 +237,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val beforeSepIndex = ctx.index repeater.accumulate(ctx.successValue.asInstanceOf[T], acc) val nextCount = count + 1 - if (whitespace ne NoWhitespace.noWhitespaceImplicit) Util.consumeWhitespace(whitespace, ctx) - - if (!ctx.isSuccess && ctx.cut) ctx.asInstanceOf[ParsingRun[Nothing]] + if (!consumeWhitespace(whitespace, ctx, false)) ctx.asInstanceOf[ParsingRun[Nothing]] else { ctx.cut = false val sep1 = sep @@ -246,9 +245,10 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val endCut = outerCut | postCut | sepCut if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) else if (ctx.isSuccess) { - if (whitespace ne NoWhitespace.noWhitespaceImplicit) Util.consumeWhitespace(whitespace, ctx) - - rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + if (!consumeWhitespace(whitespace, ctx, sepCut)) ctx.asInstanceOf[ParsingRun[Nothing]] + else { + rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + } } else { val res = @@ -264,49 +264,12 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ rec(ctx.index, 0, false, ctx.cut, null, null) } - private def aggregateMsgPostSep[V](startIndex: Int, - min: Int, - ctx: ParsingRun[Any], - parsedMsg: Msgs, - lastAgg: Msgs) = { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + s".rep($min)", - // When we fail on a sep, we collect the failure aggregate of the last - // non-sep rep body together with the failure aggregate of the sep, since - // the last non-sep rep body continuing is one of the valid ways of - // continuing the parse - ctx.failureGroupAggregate ::: lastAgg - - ) - } - - private def aggregateMsgInRep[V](startIndex: Int, - min: Int, - ctx: ParsingRun[Any], - sepMsg: Msgs, - parsedMsg: Msgs, - lastAgg: Msgs, - precut: Boolean) = { - if (sepMsg == null || precut) { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + s".rep($min)", - if (lastAgg == null) ctx.failureGroupAggregate - else ctx.failureGroupAggregate ::: lastAgg - ) - } else { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + s".rep($min)", - // When we fail on a rep body, we collect both the concatenated - // sep and failure aggregate of the rep body that we tried (because - // we backtrack past the sep on failure) as well as the failure - // aggregate of the previous rep, which we could have continued - if (lastAgg == null) Util.joinBinOp(sepMsg, parsedMsg) - else Util.joinBinOp(sepMsg, parsedMsg) ::: lastAgg - ) + private def consumeWhitespace(whitespace: fastparse.Whitespace, ctx: ParsingRun[_], extraCut: Boolean) = { + if (whitespace eq NoWhitespace.noWhitespaceImplicit) true + else { + Util.consumeWhitespace(whitespace, ctx) + if (!ctx.isSuccess && (extraCut || ctx.cut)) false + else true } } - } diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 5a214df8..65071be2 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -34,23 +34,18 @@ object MacroInlineImpls { } } else { val xLength = Expr[Int](x.length) - val checker = - '{ (string: _root_.fastparse.ParserInput, offset: _root_.scala.Int) => - ${ - x.zipWithIndex - .map { case (char, i) => '{ string.apply(offset + ${ Expr(i) }) == ${ Expr(char) } } } - .reduce[Expr[Boolean]] { case (l, r) => '{ $l && $r } } - } - } '{ - $ctx match { case ctx1 => val index = ctx1.index val end = index + $xLength val input = ctx1.input val res = - if (input.isReachable(end - 1) && ${ checker }(input, index)) { + if (input.isReachable(end - 1) && ${ + x.zipWithIndex + .map { case (char, i) => '{ input.apply(index + ${ Expr(i) }) == ${ Expr(char) } } } + .reduce[Expr[Boolean]] { case (l, r) => '{ $l && $r } } + }) { ctx1.freshSuccessUnit(end) } else { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] @@ -99,17 +94,18 @@ object MacroInlineImpls { val startIndex = ctx1.index val instrument = ctx1.instrument != null - val ctx0 = t if (instrument) { ctx1.instrument.beforeParse(name.value, startIndex) } + val ctx0 = t + if (instrument) { ctx1.instrument.afterParse(name.value, ctx0.index, ctx0.isSuccess) } if (ctx0.verboseFailures) { ctx0.aggregateMsg( startIndex, - Msgs(List(new Lazy(() => name.value))), + Msgs(new Lazy(() => name.value) :: Nil), ctx0.failureGroupAggregate, startIndex < ctx0.traceIndex ) diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 218f5e05..9ae74fb8 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -16,94 +16,150 @@ import scala.quoted.* * in the hot paths. */ object MacroRepImpls { - def repXMacro0[T: Type, V: Type]( - lhs: Expr[ParsingRun[T]], - whitespace: Null | Expr[fastparse.Whitespace], - min: Null | Expr[Int], - )(repeater: Expr[Implicits.Repeater[T, V]], ctx: Expr[ParsingRun[_]])(using quotes: Quotes): Expr[ParsingRun[V]] = { + def repMacro0[T: Type, V: Type]( + parse0: Expr[ParsingRun[T]], + sep: Expr[ParsingRun[_]], + whitespace: Expr[fastparse.Whitespace], + min: Expr[Int], + max: Expr[Int], + exactly: Expr[Int], + )(repeater0: Expr[Implicits.Repeater[T, V]], + ctx0: Expr[ParsingRun[_]])(using quotes: Quotes): Expr[ParsingRun[V]] = { import quotes.reflect.* + def getInlineExpansionValue(t: Term): Option[Int] = { + t match{ + case Inlined(a, b, c) => getInlineExpansionValue(c) + case _ => t.asExprOf[Int].value + } + } + + val staticMin0 = getInlineExpansionValue(min.asTerm) + val staticMax0 = getInlineExpansionValue(max.asTerm) + val staticExactly0 = getInlineExpansionValue(exactly.asTerm) + + val staticActualMin = staticMin0.zip(staticExactly0).map{(m, e) => if (e == -1) m else e} + val staticActualMax = staticMax0.zip(staticExactly0).map{(m, e) => if (e == -1) m else e} + '{ + val ctx = $ctx0 + val repeater = $repeater0 + val acc = repeater.initial + val actualMin = if ($exactly == -1) $min else $exactly + val actualMax = if ($exactly == -1) $max else $exactly - val ctx1 = $ctx - val repeater1 = $repeater - var originalCut = ctx1.cut - val acc = repeater1.initial - @ _root_.scala.annotation.tailrec - def rec( - startIndex: _root_.scala.Int, - count: _root_.scala.Int, - lastAgg: _root_.fastparse.internal.Msgs - ): _root_.fastparse.P[V] = { - ${ - val (endSnippet, aggregateSnippet, minCut) = min match { - case null => - ( - '{ - ctx1.freshSuccess(repeater1.result(acc), startIndex, originalCut) - }, - '{ "" }, - '{ false } - ) - case min1 => - ( - '{ - if (count < $min1) ctx1.augmentFailure(startIndex, originalCut) - else ctx1.freshSuccess(repeater1.result(acc), startIndex, originalCut) - }, - '{ if ($min1 == 0) "" else "(" + $min1 + ")" }, - '{ originalCut && (count < $min1) } - ) + def end(successIndex: Int, index: Int, count: Int, endCut: Boolean) = ${ + staticActualMin match{ + case Some(-1) => '{ ctx.freshSuccess(repeater.result(acc), successIndex, endCut) } + case _ => + '{ + if (count < actualMin) ctx.augmentFailure(index, endCut) + else ctx.freshSuccess(repeater.result(acc), successIndex, endCut) + } + } + } + + @tailrec def rec(startIndex: Int, + count: Int, + precut: Boolean, + outerCut: Boolean, + sepMsg: Msgs, + lastAgg: Msgs): ParsingRun[V] = ${ + + def consumeWhitespace(extraCut: Expr[Boolean])(x: Expr[ParsingRun[V]]) = + if whitespace.asTerm.tpe =:= TypeRepr.of[fastparse.NoWhitespace.noWhitespaceImplicit.type] + then x + else '{ + Util.consumeWhitespace($whitespace, ctx) + if (!ctx.isSuccess && ($extraCut || ctx.cut)) ctx.asInstanceOf[ParsingRun[Nothing]] + else { $x } } - '{ + val ctxCut = staticActualMin match{ + case Some(-1) => '{ precut } + case _ => '{ precut | (count < actualMin && outerCut) } + } - ctx1.cut = $minCut - $lhs + val checkMax0 = staticActualMax match{ + case Some(v) if v != 0 => '{false} + case _ => '{ count == 0 && actualMax == 0 } + } - val parsedMsg = ctx1.shortParserMsg - val parsedAgg = ctx1.failureGroupAggregate - originalCut |= ctx1.cut - if (!ctx1.isSuccess) { + '{ + ctx.cut = $ctxCut + if ($checkMax0) ctx.freshSuccess(repeater.result(acc), startIndex) + else { + $parse0 + val parsedMsg = ctx.shortParserMsg + val parsedAgg = ctx.failureGroupAggregate + val postCut = ctx.cut + val verboseFailures = ctx.verboseFailures + if (!ctx.isSuccess) { val res = - if (ctx1.cut) ctx1.asInstanceOf[_root_.fastparse.P[V]] - else $endSnippet - if (ctx1.verboseFailures) { - ctx1.aggregateMsg( - startIndex, - () => parsedMsg.render + s".rep" + $aggregateSnippet, - if (lastAgg == null) ctx1.failureGroupAggregate - else ctx1.failureGroupAggregate ::: lastAgg - ) - } + if (postCut) ctx.asInstanceOf[ParsingRun[V]] + else end(startIndex, startIndex, count, outerCut | postCut) + if (verboseFailures) Util.aggregateMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut) res } else { - val beforeSepIndex = ctx1.index - repeater1.accumulate(ctx1.successValue.asInstanceOf[T], acc) - ctx1.cut = false + val beforeSepIndex = ctx.index + repeater.accumulate(ctx.successValue.asInstanceOf[T], acc) + val nextCount = count + 1 ${ - val wsSnippet = whitespace match { - case null => '{ rec(beforeSepIndex, count + 1, parsedAgg) } - case ws => - '{ - if ($ws ne _root_.fastparse.NoWhitespace.noWhitespaceImplicit) { - _root_.fastparse.internal.Util.consumeWhitespace($ws, ctx1) - } - if (!ctx1.isSuccess && ctx1.cut) ctx1.asInstanceOf[_root_.fastparse.ParsingRun[scala.Nothing]] - else { - ctx1.cut = false - rec(beforeSepIndex, count + 1, parsedAgg) - } + val checkMax2 = staticActualMax match { + case Some(Int.MaxValue) => '{ false } + case _ => '{ nextCount == actualMax } + } + '{ + if ($checkMax2) { + val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) + if (verboseFailures) ctx.setMsg(startIndex, () => parsedMsg.render + ".rep" + (if (actualMin == 0) "" else s"(${actualMin})")) + res + } + else { + ${ + consumeWhitespace('{false})('{ + ctx.cut = false + ${ + sep match { + case '{ null } => + '{ + rec(beforeSepIndex, nextCount, false, outerCut | postCut, null, parsedAgg) + } + case _ => + '{ + val sep1 = $sep + val sepCut = ctx.cut + val endCut = outerCut | postCut | sepCut + if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) + else if (ctx.isSuccess) { + ${ + consumeWhitespace('{sepCut})('{ + rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + }) + } + } + else { + val res = + if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) + else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) + + if (verboseFailures) Util.aggregateMsgPostSep(startIndex, actualMin, ctx, parsedMsg, parsedAgg) + res + } + } + } + } + }) } + } } - wsSnippet } } } } } - rec(ctx1.index, 0, null) - } + rec(ctx.index, 0, false, ctx.cut, null, null) + } } } diff --git a/fastparse/src-3/fastparse/package.scala b/fastparse/src-3/fastparse/package.scala index 8d1d9748..aec2fc87 100644 --- a/fastparse/src-3/fastparse/package.scala +++ b/fastparse/src-3/fastparse/package.scala @@ -135,7 +135,7 @@ package object fastparse extends fastparse.SharedPackageDefs { * index of the last run. */ inline def rep[V](using repeater: Implicits.Repeater[T, V], whitespace: Whitespace, ctx: P[Any]): P[V] = - ${ MacroRepImpls.repXMacro0[T, V]('parse0, 'whitespace, null)('repeater, 'ctx) } + ${ MacroRepImpls.repMacro0[T, V]('parse0, '{null}, 'whitespace, '{0}, '{Int.MaxValue}, '{-1})('repeater, 'ctx) } /** Raw repeat operator; runs the LHS parser 0 or more times *without* * any whitespace in between, and returns @@ -143,7 +143,7 @@ package object fastparse extends fastparse.SharedPackageDefs { * index of the last run. */ inline def repX[V](using repeater: Implicits.Repeater[T, V], ctx: P[Any]): P[V] = - ${ MacroRepImpls.repXMacro0[T, V]('parse0, null, null)('repeater, 'ctx) } + ${ MacroRepImpls.repMacro0[T, V]('parse0, '{null}, '{fastparse.NoWhitespace.noWhitespaceImplicit}, '{0}, '{Int.MaxValue}, '{-1})('repeater, 'ctx) } /// ** Repeat operator; runs the LHS parser at least `min` // * times separated by the given whitespace (in implicit scope), @@ -173,19 +173,29 @@ package object fastparse extends fastparse.SharedPackageDefs { * The convenience parameter `exactly` is provided to set both `min` and * `max` to the same value. */ - def rep[V]( - min: Int = 0, - sep: => P[_] = null, - max: Int = Int.MaxValue, - exactly: Int = -1 + inline def rep[V]( + inline min: Int = 0, + inline sep: => P[_] = null, + inline max: Int = Int.MaxValue, + inline exactly: Int = -1 )(using repeater: Implicits.Repeater[T, V], whitespace: Whitespace, ctx: P[Any] - ): P[V] = - if max == Int.MaxValue && exactly == -1 - then new RepImpls[T](() => parse0).rep[V](min, sep) - else new RepImpls[T](() => parse0).rep[V](min, sep, max, exactly) + ): P[V] = ${ + MacroRepImpls.repMacro0[T, V]( + 'parse0, + 'sep, + 'whitespace, + 'min, + 'max, + 'exactly + )( + 'repeater, + 'ctx + ) + } + /** Raw repeat operator; runs the LHS parser at least `min` to at most `max` * times separated by the @@ -195,18 +205,28 @@ package object fastparse extends fastparse.SharedPackageDefs { * The convenience parameter `exactly` is provided to set both `min` and * `max` to the same value. */ - inline def repX[V, Max <: Int, Exact <: Int]( - min: Int = 0, - sep: => P[_] = null, - inline max: Max = Int.MaxValue, - inline exactly: Exact = -1 + inline def repX[V]( + inline min: Int = 0, + inline sep: => P[_] = null, + inline max: Int = Int.MaxValue, + inline exactly: Int = -1 )(implicit repeater: Implicits.Repeater[T, V], ctx: P[Any] - ): P[V] = - inline if max == Int.MaxValue && exactly == -1 - then new RepImpls[T](() => parse0).repX[V](min, sep) - else new RepImpls[T](() => parse0).repX[V](min, sep, max, exactly) + ): P[V] = ${ + MacroRepImpls.repMacro0[T, V]( + 'parse0, + 'sep, + '{fastparse.NoWhitespace.noWhitespaceImplicit}, + 'min, + 'max, + 'exactly + )( + 'repeater, + 'ctx + ) + } + /** @@ -223,7 +243,7 @@ package object fastparse extends fastparse.SharedPackageDefs { * consuming zero characters. */ def unary_!(implicit ctx: P[Any]): P[Unit] = SharedPackageDefs.unary_!(() => parse0) - end extension + /** Provides logging-related [[LogByNameOps]] implicits on [[String]]. */ implicit def LogOpsStr(parse0: String)(implicit ctx: P[Any]): fastparse.LogByNameOps[Unit] = LogByNameOps(parse0) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index ce42f65f..bcf96457 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -184,7 +184,7 @@ final class ParsingRun[+T](val input: ParserInput, def aggregateMsg(startIndex: Int, msgToSet: () => String, msgToAggregate: Msgs): Unit = { - aggregateMsg(startIndex, Msgs(List(new Lazy(msgToSet))), msgToAggregate) + aggregateMsg(startIndex, Msgs(new Lazy(msgToSet) :: Nil), msgToAggregate) } def aggregateMsg(startIndex: Int, @@ -212,15 +212,15 @@ final class ParsingRun[+T](val input: ParserInput, val f2 = new Lazy(f) if (!isSuccess){ if (index == traceIndex) failureTerminalAggregate ::= f2 - if (lastFailureMsg == null) lastFailureMsg = Msgs(List(f2)) + if (lastFailureMsg == null) lastFailureMsg = Msgs(f2 :: Nil) } - shortParserMsg = if (startIndex >= traceIndex) Msgs(List(f2)) else Msgs.empty + shortParserMsg = if (startIndex >= traceIndex) Msgs(f2 :: Nil) else Msgs.empty failureGroupAggregate = if (checkAggregate(startIndex)) shortParserMsg else Msgs.empty } def setMsg(startIndex: Int, f: () => String): Unit = { - setMsg(startIndex, Msgs(List(new Lazy(f)))) + setMsg(startIndex, Msgs(new Lazy(f) :: Nil)) } def setMsg(startIndex: Int, f: Msgs): Unit = { diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 7e0debcc..46550c35 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { @@ -152,7 +152,8 @@ object JsonnetWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { @@ -204,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 7ca52f04..ed6941aa 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -13,7 +13,7 @@ object Util { def joinBinOp(lhs: Msgs, rhs: Msgs) = if (lhs.value.isEmpty) rhs else if (rhs.value.isEmpty) lhs - else Msgs(List(new Lazy(() => lhs.render + " ~ " + rhs.render))) + else Msgs(new Lazy(() => lhs.render + " ~ " + rhs.render) :: Nil) def consumeWhitespace[V](whitespace: fastparse.Whitespace, ctx: ParsingRun[Any]) = { val oldCapturing = ctx.noDropBuffer // completely disallow dropBuffer @@ -98,6 +98,52 @@ object Util { sb.result() } + + + def aggregateMsgPostSep[V](startIndex: Int, + min: Int, + ctx: ParsingRun[Any], + parsedMsg: Msgs, + lastAgg: Msgs) = { + ctx.aggregateMsg( + startIndex, + () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), + // When we fail on a sep, we collect the failure aggregate of the last + // non-sep rep body together with the failure aggregate of the sep, since + // the last non-sep rep body continuing is one of the valid ways of + // continuing the parse + ctx.failureGroupAggregate ::: lastAgg + + ) + } + + def aggregateMsgInRep[V](startIndex: Int, + min: Int, + ctx: ParsingRun[Any], + sepMsg: Msgs, + parsedMsg: Msgs, + lastAgg: Msgs, + precut: Boolean) = { + if (sepMsg == null || precut) { + ctx.aggregateMsg( + startIndex, + () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), + if (lastAgg == null) ctx.failureGroupAggregate + else ctx.failureGroupAggregate ::: lastAgg + ) + } else { + ctx.aggregateMsg( + startIndex, + () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), + // When we fail on a rep body, we collect both the concatenated + // sep and failure aggregate of the rep body that we tried (because + // we backtrack past the sep on failure) as well as the failure + // aggregate of the previous rep, which we could have continued + if (lastAgg == null) Util.joinBinOp(sepMsg, parsedMsg) + else Util.joinBinOp(sepMsg, parsedMsg) ::: lastAgg + ) + } + } } class Lazy[T](calc0: () => T){ diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index c0e9b11a..05ca749a 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -53,6 +53,7 @@ object ExampleTests extends TestSuite{ // aggregateMsg and longAggregateMsg record all parsers // failing at the position, "a" | "b" | "c", + assert( trace.aggregateMsg == """Expected (parseEither | "c"):1:1, found "d"""", trace.longAggregateMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 4220457c..50cb6086 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -434,5 +434,50 @@ object FailureTests extends TestSuite{ parser = { implicit c => (("a".! ~ "b".!) ~ "c".!).? ~ "a".! ~ "d".! } ) } + test("whitespaceFail"){ + import ScalaWhitespace._ + test("noSeparator1") - checkOffset( + input = "a a /* */ a a /* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep } + ) + test("noSeparator2") - checkOffset( + input = "a a /* */ a a /* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep(1) } + ) + test("afterSeparator1") - checkOffset( + input = "a b a b /* */ a b a b a b/* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep(1, sep = "b") } + ) + test("afterSeparator2") - checkOffset( + input = "a b a b /* */ a b a b a b/* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep(sep = "b") } + ) + test("beforeSeparator1") - checkOffset( + input = "a b a b /* */ a b a b a /* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep(1, sep = "b") } + ) + test("beforeSeparator2") - checkOffset( + input = "a b a b /* */ a b a b a /* a a a", + expected = """"*/"""", + terminals = "\"*/\"", + label = "\"*/\"", + parser = { implicit c => "a".rep(sep = "b") } + ) + } } } diff --git a/fastparse/test/src/fastparse/JsonTests.scala b/fastparse/test/src/fastparse/JsonTests.scala index 6c5a3fab..c00ba63e 100644 --- a/fastparse/test/src/fastparse/JsonTests.scala +++ b/fastparse/test/src/fastparse/JsonTests.scala @@ -30,7 +30,6 @@ object Js { case class NamedFunction[T, V](f: T => V, name: String) extends (T => V){ def apply(t: T) = f(t) override def toString() = name - } object Json{ import fastparse._, NoWhitespace._ diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 2104e2ec..01e84569 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -983,5 +983,14 @@ object FailureTests extends TestSuite{ found = "" ) + test - checkNeg( + """object foo + |/* + | """.stripMargin, + aggregate = """"*/"""", + terminals = null, + found = "" + ) + } }