From 668cd5ac40263b9f2fb546dafbfad0b2680e6bac Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 11:34:32 +0800 Subject: [PATCH 01/49] . --- .../src-2/fastparse/internal/MacroImpls.scala | 16 ++++++++-------- .../fastparse/internal/MacroInlineImpls.scala | 16 ++++++++-------- fastparse/src/fastparse/ParsingRun.scala | 14 ++------------ fastparse/src/fastparse/SharedPackageDefs.scala | 12 ++++++------ fastparse/src/fastparse/Whitespace.scala | 6 +++--- 5 files changed, 27 insertions(+), 37 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 447e6c1c..867f40e3 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -29,7 +29,7 @@ object MacroImpls { else if (f.splice(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.setMsg(startIndex, () => "filter") res } } @@ -89,7 +89,7 @@ object MacroImpls { }else{ ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => literalized.splice) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => literalized.splice) res } @@ -119,7 +119,7 @@ object MacroImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.aggregateTerminal(index, () => literalized.splice) + ctx1.setMsg(index, () => literalized.splice) } res @@ -134,7 +134,7 @@ object MacroImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => Util.literalize(s1)) res } } @@ -429,7 +429,7 @@ object MacroImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => bracketed.splice) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => bracketed.splice) res } } @@ -570,7 +570,7 @@ object MacroImpls { else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.setMsg(startIndex, () => s"char-pred(${p0})") res } } @@ -613,7 +613,7 @@ object MacroImpls { if ($index >= $goal) $ctx1.freshSuccessUnit(index = $index) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.aggregateTerminal($start, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.setMsg($start, () => $bracketed) res } """ @@ -641,7 +641,7 @@ object MacroImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, ${min.splice})") + if (ctx0.verboseFailures) ctx0.setMsg(start, () => s"chars-while($p0, ${min.splice})") res } } diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 65071be2..61096468 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -27,7 +27,7 @@ object MacroInlineImpls { } else { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $literalized) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => $literalized) res } @@ -51,7 +51,7 @@ object MacroInlineImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.aggregateTerminal(index, () => $literalized) + ctx1.setMsg(index, () => $literalized) } res @@ -67,7 +67,7 @@ object MacroInlineImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => Util.literalize(s1)) res } } @@ -83,7 +83,7 @@ object MacroInlineImpls { else if (f(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.setMsg(startIndex, () => "filter") res } @@ -399,7 +399,7 @@ object MacroInlineImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $bracketed) + if (ctx1.verboseFailures) ctx1.setMsg(index, () => $bracketed) res } } @@ -422,7 +422,7 @@ object MacroInlineImpls { } else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.setMsg(startIndex, () => s"char-pred(${p0})") res } @@ -450,7 +450,7 @@ object MacroInlineImpls { if (index >= goal) ctx1.freshSuccessUnit(index = index) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.aggregateTerminal(start, () => $bracketed) + if (ctx1.verboseFailures) ctx1.setMsg(start, () => $bracketed) res } } @@ -465,7 +465,7 @@ object MacroInlineImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, $min)") + if (ctx0.verboseFailures) ctx0.setMsg(start, () => s"chars-while($p0, $min)") res } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index bcf96457..bf7a66d8 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -136,7 +136,7 @@ final class ParsingRun[+T](val input: ParserInput, // the parser trying to do when it failed" // // The implementation of `failureTerminalAggregate` is straightforward: we - // simply call `aggregateTerminal` in every terminal parser, which collects + // simply call `setMsg` in every terminal parser, which collects // all the messages in a big list and returns it. The implementation of // `failureGroupAggregate` is more interesting, since we need to figure out // what are the "high level" parsers that we need to list. We use the @@ -208,23 +208,13 @@ final class ParsingRun[+T](val input: ParserInput, else failureGroupAggregate = msgToAggregate } - def aggregateTerminal(startIndex: Int, f: () => String): Unit = { - val f2 = new Lazy(f) - if (!isSuccess){ - if (index == traceIndex) failureTerminalAggregate ::= f2 - if (lastFailureMsg == null) lastFailureMsg = Msgs(f2 :: Nil) - } - - shortParserMsg = if (startIndex >= traceIndex) Msgs(f2 :: Nil) else Msgs.empty - failureGroupAggregate = if (checkAggregate(startIndex)) shortParserMsg else Msgs.empty - } - def setMsg(startIndex: Int, f: () => String): Unit = { setMsg(startIndex, Msgs(new Lazy(f) :: Nil)) } def setMsg(startIndex: Int, f: Msgs): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = f + if (!isSuccess && index == traceIndex) failureTerminalAggregate ::= f.value.head shortParserMsg = if (startIndex >= traceIndex) f else Msgs.empty failureGroupAggregate = if (checkAggregate(startIndex)) shortParserMsg else Msgs.empty } diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 68a1da3b..0309c69a 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -82,7 +82,7 @@ trait SharedPackageDefs { val res = if (Util.startsWithIgnoreCase(ctx.input, s, ctx.index)) ctx.freshSuccessUnit(ctx.index + s.length) else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize(s)) + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize(s)) res } @@ -126,7 +126,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(startIndex)) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "end-of-input") + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "end-of-input") res } @@ -138,7 +138,7 @@ trait SharedPackageDefs { val res = if (startIndex == 0) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "start-of-input") + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "start-of-input") res } @@ -211,7 +211,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Unit]] else ctx.freshSuccessUnit(ctx.index + 1) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "any-character") res } @@ -227,7 +227,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Char]] else ctx.freshSuccess(ctx.input(ctx.index), ctx.index + 1) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "any-character") res } @@ -262,7 +262,7 @@ object SharedPackageDefs{ if (res.isSuccess) ctx.freshSuccess(ctx.successValue) else ctx.freshFailure(oldIndex) - if (ctx.verboseFailures) ctx.aggregateTerminal(oldIndex, () => msg) + if (ctx.verboseFailures) ctx.setMsg(oldIndex, () => msg) res2.asInstanceOf[P[T]] } diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 46550c35..299b9b44 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) res } } else { @@ -153,7 +153,7 @@ object JsonnetWhitespace{ ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) res } } else { @@ -205,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) res } } else { From 36590a71a80632449e270bfed8679c4f5f90704a Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 11:42:13 +0800 Subject: [PATCH 02/49] converge aggregateMsg and setMsg --- .../src-2/fastparse/internal/MacroImpls.scala | 16 +++++++-------- .../fastparse/internal/MacroInlineImpls.scala | 16 +++++++-------- fastparse/src/fastparse/ParsingRun.scala | 20 +++++++++++++------ .../src/fastparse/SharedPackageDefs.scala | 12 +++++------ fastparse/src/fastparse/Whitespace.scala | 6 +++--- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 867f40e3..447e6c1c 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -29,7 +29,7 @@ object MacroImpls { else if (f.splice(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.setMsg(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") res } } @@ -89,7 +89,7 @@ object MacroImpls { }else{ ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.setMsg(index, () => literalized.splice) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => literalized.splice) res } @@ -119,7 +119,7 @@ object MacroImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.setMsg(index, () => literalized.splice) + ctx1.aggregateTerminal(index, () => literalized.splice) } res @@ -134,7 +134,7 @@ object MacroImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.setMsg(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) res } } @@ -429,7 +429,7 @@ object MacroImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.setMsg(index, () => bracketed.splice) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => bracketed.splice) res } } @@ -570,7 +570,7 @@ object MacroImpls { else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.setMsg(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") res } } @@ -613,7 +613,7 @@ object MacroImpls { if ($index >= $goal) $ctx1.freshSuccessUnit(index = $index) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.setMsg($start, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.aggregateTerminal($start, () => $bracketed) res } """ @@ -641,7 +641,7 @@ object MacroImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.setMsg(start, () => s"chars-while($p0, ${min.splice})") + if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, ${min.splice})") res } } diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 61096468..65071be2 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -27,7 +27,7 @@ object MacroInlineImpls { } else { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.setMsg(index, () => $literalized) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $literalized) res } @@ -51,7 +51,7 @@ object MacroInlineImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.setMsg(index, () => $literalized) + ctx1.aggregateTerminal(index, () => $literalized) } res @@ -67,7 +67,7 @@ object MacroInlineImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.setMsg(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) res } } @@ -83,7 +83,7 @@ object MacroInlineImpls { else if (f(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.setMsg(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") res } @@ -399,7 +399,7 @@ object MacroInlineImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.setMsg(index, () => $bracketed) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $bracketed) res } } @@ -422,7 +422,7 @@ object MacroInlineImpls { } else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.setMsg(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") res } @@ -450,7 +450,7 @@ object MacroInlineImpls { if (index >= goal) ctx1.freshSuccessUnit(index = index) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.setMsg(start, () => $bracketed) + if (ctx1.verboseFailures) ctx1.aggregateTerminal(start, () => $bracketed) res } } @@ -465,7 +465,7 @@ object MacroInlineImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.setMsg(start, () => s"chars-while($p0, $min)") + if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, $min)") res } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index bf7a66d8..f5640308 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -136,7 +136,7 @@ final class ParsingRun[+T](val input: ParserInput, // the parser trying to do when it failed" // // The implementation of `failureTerminalAggregate` is straightforward: we - // simply call `setMsg` in every terminal parser, which collects + // simply call `aggregateTerminal` in every terminal parser, which collects // all the messages in a big list and returns it. The implementation of // `failureGroupAggregate` is more interesting, since we need to figure out // what are the "high level" parsers that we need to list. We use the @@ -198,14 +198,20 @@ final class ParsingRun[+T](val input: ParserInput, forceAggregate: Boolean): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet - shortParserMsg = msgToSet // There are two cases when aggregating: either we stomp over the entire // existing aggregation with `msgToSet`, or we preserve it (with possible // additions) with `msgToAggregate`. - if (checkAggregate(startIndex) && !forceAggregate) failureGroupAggregate = msgToSet - else failureGroupAggregate = msgToAggregate + failureGroupAggregate = + if (checkAggregate(startIndex) && !forceAggregate) msgToSet + else msgToAggregate + } + + def aggregateTerminal(startIndex: Int, f: () => String): Unit = { + val f2 = new Lazy(f) + if (!isSuccess && index == traceIndex) failureTerminalAggregate ::= f2 + setMsg(startIndex, Msgs(f2 :: Nil)) } def setMsg(startIndex: Int, f: () => String): Unit = { @@ -214,9 +220,11 @@ final class ParsingRun[+T](val input: ParserInput, def setMsg(startIndex: Int, f: Msgs): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = f - if (!isSuccess && index == traceIndex) failureTerminalAggregate ::= f.value.head + shortParserMsg = if (startIndex >= traceIndex) f else Msgs.empty - failureGroupAggregate = if (checkAggregate(startIndex)) shortParserMsg else Msgs.empty + failureGroupAggregate = + if (checkAggregate(startIndex)) shortParserMsg + else Msgs.empty } /** diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 0309c69a..68a1da3b 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -82,7 +82,7 @@ trait SharedPackageDefs { val res = if (Util.startsWithIgnoreCase(ctx.input, s, ctx.index)) ctx.freshSuccessUnit(ctx.index + s.length) else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize(s)) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize(s)) res } @@ -126,7 +126,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(startIndex)) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "end-of-input") + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "end-of-input") res } @@ -138,7 +138,7 @@ trait SharedPackageDefs { val res = if (startIndex == 0) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "start-of-input") + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "start-of-input") res } @@ -211,7 +211,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Unit]] else ctx.freshSuccessUnit(ctx.index + 1) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") res } @@ -227,7 +227,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Char]] else ctx.freshSuccess(ctx.input(ctx.index), ctx.index + 1) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") res } @@ -262,7 +262,7 @@ object SharedPackageDefs{ if (res.isSuccess) ctx.freshSuccess(ctx.successValue) else ctx.freshFailure(oldIndex) - if (ctx.verboseFailures) ctx.setMsg(oldIndex, () => msg) + if (ctx.verboseFailures) ctx.aggregateTerminal(oldIndex, () => msg) res2.asInstanceOf[P[T]] } diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 299b9b44..46550c35 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { @@ -153,7 +153,7 @@ object JsonnetWhitespace{ ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { @@ -205,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.setMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) res } } else { From 003baff1c7c06d419cfceaef7bb38ca29366bf6b Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 11:50:13 +0800 Subject: [PATCH 03/49] more convergence --- fastparse/src/fastparse/ParsingRun.scala | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index f5640308..c9083974 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -195,16 +195,20 @@ final class ParsingRun[+T](val input: ParserInput, def aggregateMsg(startIndex: Int, msgToSet: Msgs, msgToAggregate: Msgs, - forceAggregate: Boolean): Unit = { + forceAggregate: Boolean, + setShortMsgOnlyIfBeyondTraceIndex: Boolean = false): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet - shortParserMsg = msgToSet + + shortParserMsg = + if (!setShortMsgOnlyIfBeyondTraceIndex || startIndex >= traceIndex) msgToSet + else Msgs.empty // There are two cases when aggregating: either we stomp over the entire // existing aggregation with `msgToSet`, or we preserve it (with possible // additions) with `msgToAggregate`. failureGroupAggregate = - if (checkAggregate(startIndex) && !forceAggregate) msgToSet + if (checkAggregate(startIndex) && !forceAggregate) shortParserMsg else msgToAggregate } @@ -218,13 +222,14 @@ final class ParsingRun[+T](val input: ParserInput, setMsg(startIndex, Msgs(new Lazy(f) :: Nil)) } - def setMsg(startIndex: Int, f: Msgs): Unit = { - if (!isSuccess && lastFailureMsg == null) lastFailureMsg = f - - shortParserMsg = if (startIndex >= traceIndex) f else Msgs.empty - failureGroupAggregate = - if (checkAggregate(startIndex)) shortParserMsg - else Msgs.empty + def setMsg(startIndex: Int, msgToSet: Msgs): Unit = { + aggregateMsg( + startIndex, + msgToSet, + Msgs.empty, + false, + true + ) } /** From 34b1441d45b099e34b1a150b59b63c749b399b4c Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 15:55:03 +0800 Subject: [PATCH 04/49] . --- fastparse/src/fastparse/internal/Util.scala | 44 +++++++-------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index ed6941aa..cdddfa64 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -105,16 +105,7 @@ object Util { ctx: ParsingRun[Any], parsedMsg: Msgs, lastAgg: Msgs) = { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), - // When we fail on a sep, we collect the failure aggregate of the last - // non-sep rep body together with the failure aggregate of the sep, since - // the last non-sep rep body continuing is one of the valid ways of - // continuing the parse - ctx.failureGroupAggregate ::: lastAgg - - ) + aggregateMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) } def aggregateMsgInRep[V](startIndex: Int, @@ -124,25 +115,20 @@ object Util { parsedMsg: Msgs, lastAgg: Msgs, precut: Boolean) = { - if (sepMsg == null || precut) { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), - if (lastAgg == null) ctx.failureGroupAggregate - else ctx.failureGroupAggregate ::: lastAgg - ) - } else { - ctx.aggregateMsg( - startIndex, - () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), - // When we fail on a rep body, we collect both the concatenated - // sep and failure aggregate of the rep body that we tried (because - // we backtrack past the sep on failure) as well as the failure - // aggregate of the previous rep, which we could have continued - if (lastAgg == null) Util.joinBinOp(sepMsg, parsedMsg) - else Util.joinBinOp(sepMsg, parsedMsg) ::: lastAgg - ) - } + // When we fail on a rep body, we collect both the concatenated + // sep and failure aggregate of the rep body that we tried (because + // we backtrack past the sep on failure) as well as the failure + // aggregate of the previous rep, which we could have continued + val newAgg = + if (sepMsg == null || precut) ctx.failureGroupAggregate + else Util.joinBinOp(sepMsg, parsedMsg) + + ctx.aggregateMsg( + startIndex, + () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), + if (lastAgg == null) newAgg + else newAgg ::: lastAgg + ) } } From f3ff21bb56982d40c20c35beeda235811754e828 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 16:12:40 +0800 Subject: [PATCH 05/49] . --- fastparse/src/fastparse/ParsingRun.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index c9083974..63cdd100 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -196,13 +196,11 @@ final class ParsingRun[+T](val input: ParserInput, msgToSet: Msgs, msgToAggregate: Msgs, forceAggregate: Boolean, - setShortMsgOnlyIfBeyondTraceIndex: Boolean = false): Unit = { + setShortMsg: Boolean = true): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet - shortParserMsg = - if (!setShortMsgOnlyIfBeyondTraceIndex || startIndex >= traceIndex) msgToSet - else Msgs.empty + shortParserMsg = if (setShortMsg) msgToSet else Msgs.empty // There are two cases when aggregating: either we stomp over the entire // existing aggregation with `msgToSet`, or we preserve it (with possible @@ -228,7 +226,7 @@ final class ParsingRun[+T](val input: ParserInput, msgToSet, Msgs.empty, false, - true + startIndex >= traceIndex ) } From 6c21dcb409034033e36747dbed2c0a0c35099a47 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 16:17:06 +0800 Subject: [PATCH 06/49] . --- fastparse/src/fastparse/ParsingRun.scala | 24 ++++----------------- fastparse/src/fastparse/internal/Util.scala | 3 +++ 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 63cdd100..1644ce87 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -181,21 +181,10 @@ final class ParsingRun[+T](val input: ParserInput, // to force it to expose it's internals when it's range covers the `traceIndex` // but it isn't an exact match - def aggregateMsg(startIndex: Int, - msgToSet: () => String, - msgToAggregate: Msgs): Unit = { - aggregateMsg(startIndex, Msgs(new Lazy(msgToSet) :: Nil), msgToAggregate) - } - - def aggregateMsg(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs): Unit = { - aggregateMsg(startIndex, msgToSet, msgToAggregate, false) - } def aggregateMsg(startIndex: Int, msgToSet: Msgs, msgToAggregate: Msgs, - forceAggregate: Boolean, + forceAggregate: Boolean = false, setShortMsg: Boolean = true): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet @@ -210,14 +199,9 @@ final class ParsingRun[+T](val input: ParserInput, else msgToAggregate } - def aggregateTerminal(startIndex: Int, f: () => String): Unit = { - val f2 = new Lazy(f) - if (!isSuccess && index == traceIndex) failureTerminalAggregate ::= f2 - setMsg(startIndex, Msgs(f2 :: Nil)) - } - - def setMsg(startIndex: Int, f: () => String): Unit = { - setMsg(startIndex, Msgs(new Lazy(f) :: Nil)) + def aggregateTerminal(startIndex: Int, f: Msgs): Unit = { + if (!isSuccess && index == traceIndex) failureTerminalAggregate :::= f + setMsg(startIndex, f) } def setMsg(startIndex: Int, msgToSet: Msgs): Unit = { diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index cdddfa64..b3728a00 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -180,6 +180,9 @@ final class CompactTrieNode(source: TrieNode){ } object Msgs{ val empty = Msgs(Nil) + implicit def fromFunction(msgToSet: () => String): Msgs = { + Msgs(new Lazy(msgToSet) :: Nil) + } } case class Msgs(value: List[Lazy[String]]){ def :::(other: Msgs) = Msgs(other.value ::: value) From 550e64aaeb3e66e605958d6a83861253d2286865 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 16:40:23 +0800 Subject: [PATCH 07/49] improve compile time handling of default value sep --- .../fastparse/internal/MacroRepImpls.scala | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 9ae74fb8..8fc2cf90 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -27,19 +27,26 @@ object MacroRepImpls { ctx0: Expr[ParsingRun[_]])(using quotes: Quotes): Expr[ParsingRun[V]] = { import quotes.reflect.* - def getInlineExpansionValue(t: Term): Option[Int] = { + def getInlineExpansionValue[T](t: Term): Term = { t match{ case Inlined(a, b, c) => getInlineExpansionValue(c) - case _ => t.asExprOf[Int].value + case Typed(a, b) => getInlineExpansionValue(a) + case _ => t } } - val staticMin0 = getInlineExpansionValue(min.asTerm) - val staticMax0 = getInlineExpansionValue(max.asTerm) - val staticExactly0 = getInlineExpansionValue(exactly.asTerm) + val staticMin0 = getInlineExpansionValue[Int](min.asTerm).asExprOf[Int] + val staticMax0 = getInlineExpansionValue[Int](max.asTerm).asExprOf[Int] + val staticExactly0 = getInlineExpansionValue[Int](exactly.asTerm).asExprOf[Int] - val staticActualMin = staticMin0.zip(staticExactly0).map{(m, e) => if (e == -1) m else e} - val staticActualMax = staticMax0.zip(staticExactly0).map{(m, e) => if (e == -1) m else e} + val staticActualMin = staticExactly0 match{ + case '{-1} => staticMin0.value + case _ => staticExactly0.value + } + val staticActualMax = staticExactly0 match{ + case '{-1} => staticMax0.value + case _ => staticExactly0.value + } '{ val ctx = $ctx0 @@ -120,7 +127,7 @@ object MacroRepImpls { consumeWhitespace('{false})('{ ctx.cut = false ${ - sep match { + getInlineExpansionValue(sep.asTerm).asExpr match { case '{ null } => '{ rec(beforeSepIndex, nextCount, false, outerCut | postCut, null, parsedAgg) @@ -130,8 +137,7 @@ object MacroRepImpls { val sep1 = $sep val sepCut = ctx.cut val endCut = outerCut | postCut | sepCut - if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) - else if (ctx.isSuccess) { + if (ctx.isSuccess) { ${ consumeWhitespace('{sepCut})('{ rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) From 08503af533929f0b1080bbd81b2efddf4cb612cf Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 16:47:32 +0800 Subject: [PATCH 08/49] mroe reuse --- .../fastparse/internal/MacroRepImpls.scala | 19 +++++++++++-------- .../fastparse/internal/MacroInlineImpls.scala | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala index 962383d1..12efc34f 100644 --- a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala @@ -81,14 +81,17 @@ object MacroRepImpls{ val res = if ($ctx1.cut) $ctx1.asInstanceOf[_root_.fastparse.P[${c.weakTypeOf[V]}]] else $endSnippet - if ($ctx1.verboseFailures) { - $ctx1.aggregateMsg( - $startIndex, - () => $parsedMsg.render + s".rep" + $aggregateSnippet, - if ($lastAgg == null) $ctx1.failureGroupAggregate - else $ctx1.failureGroupAggregate ::: $lastAgg - ) - } + + if ($ctx1.verboseFailures) _root_.fastparse.internal.Util.aggregateMsgInRep( + $startIndex, + ${min.getOrElse(q"0")}, + $ctx1, + _root_.fastparse.internal.Msgs.empty, + $parsedMsg, + $lastAgg, + true + ) + res }else { val $beforeSepIndex = $ctx1.index diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 65071be2..68504c81 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -190,7 +190,7 @@ object MacroInlineImpls { if (ctx1.verboseFailures) ctx1.aggregateMsg( preLhsIndex, - _root_.fastparse.internal.Util.joinBinOp(lhsMsg, rhsMsg), + Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, // We override the failureGroupAggregate to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to @@ -208,7 +208,7 @@ object MacroInlineImpls { if (ws.asTerm.tpe =:= TypeRepr.of[fastparse.NoWhitespace.noWhitespaceImplicit.type]) rhsSnippet else { '{ - _root_.fastparse.internal.Util.consumeWhitespace($ws, ctx1) + Util.consumeWhitespace($ws, ctx1) if (ctx1.isSuccess) $rhsSnippet else ctx1 } @@ -217,7 +217,7 @@ object MacroInlineImpls { guardedRhs } } - }.asInstanceOf[_root_.fastparse.ParsingRun[R]] + }.asInstanceOf[ParsingRun[R]] } } From b59f0d32fac21a170f719dfca73d256e389aa7f5 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 17:45:01 +0800 Subject: [PATCH 09/49] consolidate down to just two error reporting functions --- cssparse/test/src-jvm/cssparse/TestUtil.scala | 2 +- .../src-2/fastparse/internal/MacroImpls.scala | 32 ++++++++--------- .../fastparse/internal/MacroRepImpls.scala | 2 +- .../src-2/fastparse/internal/RepImpls.scala | 22 ++++++------ .../fastparse/internal/MacroInlineImpls.scala | 35 +++++++++--------- .../fastparse/internal/MacroRepImpls.scala | 6 ++-- fastparse/src/fastparse/Parsed.scala | 12 +++---- fastparse/src/fastparse/ParsingRun.scala | 36 +++++++++++-------- .../src/fastparse/SharedPackageDefs.scala | 24 ++++++------- fastparse/src/fastparse/Whitespace.scala | 6 ++-- fastparse/src/fastparse/internal/Util.scala | 36 +++++++++++-------- .../fastparse/CustomWhitespaceMathTests.scala | 2 +- .../test/src/fastparse/ExampleTests.scala | 28 +++++++-------- .../test/src/fastparse/IndentationTests.scala | 2 +- fastparse/test/src/fastparse/Main.scala | 2 +- fastparse/test/src/fastparse/MathTests.scala | 4 +-- .../src/fastparse/WhitespaceMathTests.scala | 2 +- scalaparse/test/src/scalaparse/TestUtil.scala | 25 +++++++++---- .../src/scalaparse/unit/FailureTests.scala | 2 +- 19 files changed, 151 insertions(+), 129 deletions(-) diff --git a/cssparse/test/src-jvm/cssparse/TestUtil.scala b/cssparse/test/src-jvm/cssparse/TestUtil.scala index fdd3bb3d..05bec257 100644 --- a/cssparse/test/src-jvm/cssparse/TestUtil.scala +++ b/cssparse/test/src-jvm/cssparse/TestUtil.scala @@ -9,7 +9,7 @@ object TestUtil { def checkParsed(input: String, res: Parsed[Ast.RuleList]) = { res match { case f: Parsed.Failure => - throw new Exception(tag + "\n" + input + "\n" + f.trace().longAggregateMsg) + throw new Exception(tag + "\n" + input + "\n" + f.trace().longreportParseMsg) case s: Parsed.Success[Ast.RuleList] => val inputLength = input.length val index = s.index diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 447e6c1c..619da612 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -29,7 +29,7 @@ object MacroImpls { else if (f.splice(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(startIndex, () => "filter") res } } @@ -52,9 +52,9 @@ object MacroImpls { ctx1.instrument.afterParse(name.splice.value, ctx0.index, ctx0.isSuccess) } if (ctx0.verboseFailures) { - ctx0.aggregateMsg( + ctx0.reportParseMsg( startIndex, - Msgs(new Lazy(() => name.splice.value) :: Nil), + () => name.splice.value, ctx0.failureGroupAggregate, startIndex < ctx0.traceIndex ) @@ -89,7 +89,7 @@ object MacroImpls { }else{ ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => literalized.splice) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => literalized.splice) res } @@ -119,7 +119,7 @@ object MacroImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.aggregateTerminal(index, () => literalized.splice) + ctx1.reportTerminalParseMsg(index, () => literalized.splice) } res @@ -134,7 +134,7 @@ object MacroImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => Util.literalize(s1)) res } } @@ -242,7 +242,7 @@ object MacroImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.aggregateMsg(startPos, lhsMsg, lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg, lhsAggregate) ctx5.cut = false other.splice @@ -251,7 +251,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.aggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -350,7 +350,7 @@ object MacroImpls { ) } - val bracketed = "StringIn(" + literals.map(Util.literalize(_)).mkString(", ") + ")" + val bracketed = literals.map(Util.literalize(_)).toList val res = q""" $ctx match{ case $ctx1 => @@ -363,7 +363,7 @@ object MacroImpls { val res = if ($output != -1) $ctx1.freshSuccessUnit(index = $output) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.setMsg($index, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($index, () => $bracketed) res } """ @@ -429,7 +429,7 @@ object MacroImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => bracketed.splice) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => bracketed.splice) res } } @@ -488,7 +488,7 @@ object MacroImpls { ) } - if ($ctx1.verboseFailures) $ctx1.aggregateMsg( + if ($ctx1.verboseFailures) $ctx1.reportParseMsg( $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, @@ -570,7 +570,7 @@ object MacroImpls { else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(startIndex, () => s"char-pred(${p0})") res } } @@ -613,7 +613,7 @@ object MacroImpls { if ($index >= $goal) $ctx1.freshSuccessUnit(index = $index) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.aggregateTerminal($start, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($start, () => $bracketed) res } """ @@ -641,7 +641,7 @@ object MacroImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, ${min.splice})") + if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(start, () => s"chars-while($p0, ${min.splice})") res } } @@ -695,7 +695,7 @@ object MacroImpls { val msg = ctx1.shortParserMsg val agg = ctx1.failureGroupAggregate if (!postSuccess){ - ctx1.aggregateMsg(startPos, () => msg.render + ".?", agg) + ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) } } res diff --git a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala index 12efc34f..03fea383 100644 --- a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala @@ -82,7 +82,7 @@ object MacroRepImpls{ if ($ctx1.cut) $ctx1.asInstanceOf[_root_.fastparse.P[${c.weakTypeOf[V]}]] else $endSnippet - if ($ctx1.verboseFailures) _root_.fastparse.internal.Util.aggregateMsgInRep( + if ($ctx1.verboseFailures) _root_.fastparse.internal.Util.reportParseMsgInRep( $startIndex, ${min.getOrElse(q"0")}, $ctx1, diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index acdd2f5b..7e3835fb 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -2,7 +2,7 @@ package fastparse.internal import fastparse.{Implicits, NoWhitespace, ParsingRun} -import Util.{aggregateMsgInRep, aggregateMsgPostSep} +import Util.{reportParseMsgInRep, reportParseMsgPostSep} import scala.annotation.tailrec @@ -41,7 +41,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) aggregateMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) res }else { val beforeSepIndex = ctx.index @@ -49,7 +49,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val nextCount = count + 1 if (nextCount == actualMax) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.setMsg(startIndex, () => parsedMsg.render + ".repX" + (if(min == 0) "" else s"($min)")) + if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".repX" + (if(min == 0) "" else s"($min)")) res } else { @@ -64,7 +64,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) + if (verboseFailures) reportParseMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) res } } @@ -102,7 +102,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) aggregateMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) res }else { val beforeSepIndex = ctx.index @@ -119,7 +119,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) + if (verboseFailures) reportParseMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) res } } @@ -163,7 +163,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) aggregateMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) res } else { val beforeSepIndex = ctx.index @@ -171,7 +171,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val nextCount = count + 1 if (nextCount == actualMax) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.setMsg(startIndex, () => parsedMsg.render + ".rep" + (if(min == 0) "" else s"($min)")) + if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".rep" + (if(min == 0) "" else s"($min)")) res } else if (!consumeWhitespace(whitespace, ctx, false)) ctx.asInstanceOf[ParsingRun[Nothing]] @@ -192,7 +192,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) + if (verboseFailures) reportParseMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) res } } @@ -231,7 +231,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) aggregateMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) res }else{ val beforeSepIndex = ctx.index @@ -255,7 +255,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - if (verboseFailures) aggregateMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) + if (verboseFailures) reportParseMsgPostSep(startIndex, min, ctx, parsedMsg, parsedAgg) res } } diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 68504c81..1405bbb9 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -27,7 +27,7 @@ object MacroInlineImpls { } else { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $literalized) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => $literalized) res } @@ -51,7 +51,7 @@ object MacroInlineImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.aggregateTerminal(index, () => $literalized) + ctx1.reportTerminalParseMsg(index, () => $literalized) } res @@ -67,7 +67,7 @@ object MacroInlineImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => Util.literalize(s1)) res } } @@ -83,7 +83,7 @@ object MacroInlineImpls { else if (f(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.aggregateTerminal(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(startIndex, () => "filter") res } @@ -103,9 +103,9 @@ object MacroInlineImpls { ctx1.instrument.afterParse(name.value, ctx0.index, ctx0.isSuccess) } if (ctx0.verboseFailures) { - ctx0.aggregateMsg( + ctx0.reportParseMsg( startIndex, - Msgs(new Lazy(() => name.value) :: Nil), + () => name.value, ctx0.failureGroupAggregate, startIndex < ctx0.traceIndex ) @@ -188,7 +188,7 @@ object MacroInlineImpls { ) } - if (ctx1.verboseFailures) ctx1.aggregateMsg( + if (ctx1.verboseFailures) ctx1.reportParseMsg( preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, @@ -247,7 +247,7 @@ object MacroInlineImpls { val msg = ctx1.shortParserMsg val agg = ctx1.failureGroupAggregate if (!postSuccess) { - ctx1.aggregateMsg(startPos, () => msg.render + ".?", agg) + ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) } } res @@ -310,7 +310,7 @@ object MacroInlineImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.aggregateMsg(startPos, lhsMsg, lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg, lhsAggregate) ctx5.cut = false other @@ -320,7 +320,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) - ctx5.aggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) + ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -399,7 +399,7 @@ object MacroInlineImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.aggregateTerminal(index, () => $bracketed) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => $bracketed) res } } @@ -422,7 +422,7 @@ object MacroInlineImpls { } else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.aggregateTerminal(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(startIndex, () => s"char-pred(${p0})") res } @@ -450,7 +450,7 @@ object MacroInlineImpls { if (index >= goal) ctx1.freshSuccessUnit(index = index) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.aggregateTerminal(start, () => $bracketed) + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(start, () => $bracketed) res } } @@ -465,7 +465,7 @@ object MacroInlineImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.aggregateTerminal(start, () => s"chars-while($p0, $min)") + if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(start, () => s"chars-while($p0, $min)") res } @@ -540,12 +540,9 @@ object MacroInlineImpls { val res = if (output != -1) ctx1.freshSuccessUnit(output) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.setMsg( + if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg( index, - () => - ${ - Expr("StringIn(" + literals.map(Util.literalize(_)).mkString(", ") + ")") - } + () => ${ Expr(literals.map(Util.literalize(_)).toList) } ) res } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 8fc2cf90..246f7dd9 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -105,7 +105,7 @@ object MacroRepImpls { val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) Util.aggregateMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) Util.reportParseMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut) res } else { val beforeSepIndex = ctx.index @@ -119,7 +119,7 @@ object MacroRepImpls { '{ if ($checkMax2) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.setMsg(startIndex, () => parsedMsg.render + ".rep" + (if (actualMin == 0) "" else s"(${actualMin})")) + if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".rep" + (if (actualMin == 0) "" else s"(${actualMin})")) res } else { @@ -149,7 +149,7 @@ object MacroRepImpls { if (sepCut) ctx.augmentFailure(beforeSepIndex, endCut) else end(beforeSepIndex, beforeSepIndex, nextCount, endCut) - if (verboseFailures) Util.aggregateMsgPostSep(startIndex, actualMin, ctx, parsedMsg, parsedAgg) + if (verboseFailures) Util.reportParseMsgPostSep(startIndex, actualMin, ctx, parsedMsg, parsedAgg) res } } diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index a1f35569..f54a795f 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -75,7 +75,7 @@ object Parsed{ Failure.formatMsg(extra.input, extra.stack ++ List(label -> index), index) } else throw new Exception( "`.longMsg` requires the parser to be run with `verboseFailures = true`, " + - "or to be called via `.trace().longMsg` or `.trace().longAggregateMsg`" + "or to be called via `.trace().longMsg` or `.trace().longreportParseMsg`" ) } @@ -180,12 +180,12 @@ object Parsed{ def groupAggregateString = groups.render @deprecated("Use .msg instead") - def trace = aggregateMsg + def trace = reportParseMsg /** * Displays the short failure message excluding the parse stack. This shows * the last parser which failed causing the parse to fail. Note that this * does not include other parsers which may have failed earlier; see [[terminalsMsg]] - * and [[aggregateMsg]] for more detailed errors + * and [[reportParseMsg]] for more detailed errors */ def msg = failure.msg /** @@ -203,7 +203,7 @@ object Parsed{ * at the failure index. This gives you a good high-level overview of what * the parser expected, at the cost */ - def aggregateMsg = Failure.formatMsg(input, List(groupAggregateString -> index), index) + def reportParseMsg = Failure.formatMsg(input, List(groupAggregateString -> index), index) /** * A version of [[msg]] that includes the parse stack @@ -216,8 +216,8 @@ object Parsed{ def longTerminalsMsg = Failure.formatMsg(input, stack ++ Seq(terminalAggregateString -> index), index) /** - * A version of [[aggregateMsg]] that includes the parse stack + * A version of [[reportParseMsg]] that includes the parse stack */ - def longAggregateMsg = Failure.formatMsg(input, stack ++ Seq(groupAggregateString -> index), index) + def longreportParseMsg = Failure.formatMsg(input, stack ++ Seq(groupAggregateString -> index), index) } } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 1644ce87..204d51f1 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -11,12 +11,12 @@ import fastparse.internal.{Instrument, Lazy, Msgs, Util} * There are a few patterns that let us program with these mutable variables * in a sort-of-pure-functional way: * - test - If a parser that wishes to ignore changes to a field within their child + * - If a parser that wishes to ignore changes to a field within their child * parsers, a common pattern is to save the value of the field before the * wrapped parser runs, and then re-set the field. e.g. this can be used to * backtrack [[index]] after a lookahead parser finishes * - test - If a parser wants to read the value of the field "returned" by multiple + * - If a parser wants to read the value of the field "returned" by multiple * child parsers, make sure to read the field into a local variable after * each child parser is complete to make sure the value you want from an * earlier child isn't stomped over by a later child @@ -136,7 +136,7 @@ final class ParsingRun[+T](val input: ParserInput, // the parser trying to do when it failed" // // The implementation of `failureTerminalAggregate` is straightforward: we - // simply call `aggregateTerminal` in every terminal parser, which collects + // simply call `reportTerminalParseMsg` in every terminal parser, which collects // all the messages in a big list and returns it. The implementation of // `failureGroupAggregate` is more interesting, since we need to figure out // what are the "high level" parsers that we need to list. We use the @@ -181,11 +181,14 @@ final class ParsingRun[+T](val input: ParserInput, // to force it to expose it's internals when it's range covers the `traceIndex` // but it isn't an exact match - def aggregateMsg(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs, - forceAggregate: Boolean = false, - setShortMsg: Boolean = true): Unit = { + /** + * Called by non-terminal parsers after completion, success or failure + */ + def reportParseMsg(startIndex: Int, + msgToSet: Msgs, + msgToAggregate: Msgs, + forceAggregate: Boolean = false, + setShortMsg: Boolean = true): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet @@ -199,13 +202,16 @@ final class ParsingRun[+T](val input: ParserInput, else msgToAggregate } - def aggregateTerminal(startIndex: Int, f: Msgs): Unit = { - if (!isSuccess && index == traceIndex) failureTerminalAggregate :::= f - setMsg(startIndex, f) - } - - def setMsg(startIndex: Int, msgToSet: Msgs): Unit = { - aggregateMsg( + /** + * Called by any terminal parser; these are the smallest parsers that a user + * may care about, e.g. individual strings or characters, and will be stored + * in the `failureTerminalAggregate` in case a user wants to know what could + * have been placed at the failure point to let the parse progress + */ + def reportTerminalParseMsg(startIndex: Int, + msgToSet: Msgs): Unit = { + if (!isSuccess && index == traceIndex) failureTerminalAggregate :::= msgToSet + reportParseMsg( startIndex, msgToSet, Msgs.empty, diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 68a1da3b..634b1e22 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -82,7 +82,7 @@ trait SharedPackageDefs { val res = if (Util.startsWithIgnoreCase(ctx.input, s, ctx.index)) ctx.freshSuccessUnit(ctx.index + s.length) else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize(s)) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize(s)) res } @@ -106,7 +106,7 @@ trait SharedPackageDefs { else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { ctx.failureGroupAggregate = Msgs.empty - ctx.setMsg(startPos, () => + ctx.reportTerminalParseMsg(startPos, () => msg match{ case Seq(x) => s"&(${msg.render})" case xs => s"&${msg.render}" @@ -126,7 +126,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(startIndex)) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "end-of-input") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "end-of-input") res } @@ -138,7 +138,7 @@ trait SharedPackageDefs { val res = if (startIndex == 0) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "start-of-input") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "start-of-input") res } @@ -167,7 +167,7 @@ trait SharedPackageDefs { */ def Pass(implicit ctx: P[_]): P[Unit] = { val res = ctx.freshSuccessUnit() - if (ctx.verboseFailures) ctx.setMsg(ctx.index, () => "Pass") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Pass") res } @@ -177,7 +177,7 @@ trait SharedPackageDefs { */ def Pass[T](v: T)(implicit ctx: P[_]): P[T] = { val res = ctx.freshSuccess(v) - if (ctx.verboseFailures) ctx.setMsg(ctx.index, () => "Pass") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Pass") res } @@ -186,7 +186,7 @@ trait SharedPackageDefs { */ def Fail(implicit ctx: P[_]): P[Nothing] = { val res = ctx.freshFailure() - if (ctx.verboseFailures) ctx.setMsg(ctx.index, () => "fail") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "fail") res } @@ -198,7 +198,7 @@ trait SharedPackageDefs { */ def Index(implicit ctx: P[_]): P[Int] = { val res = ctx.freshSuccess(ctx.index) - if (ctx.verboseFailures) ctx.setMsg(ctx.index, () => "Index") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Index") res } @@ -211,7 +211,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Unit]] else ctx.freshSuccessUnit(ctx.index + 1) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-character") res } @@ -227,7 +227,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Char]] else ctx.freshSuccess(ctx.input(ctx.index), ctx.index + 1) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-character") res } @@ -262,7 +262,7 @@ object SharedPackageDefs{ if (res.isSuccess) ctx.freshSuccess(ctx.successValue) else ctx.freshFailure(oldIndex) - if (ctx.verboseFailures) ctx.aggregateTerminal(oldIndex, () => msg) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(oldIndex, () => msg) res2.asInstanceOf[P[T]] } @@ -284,7 +284,7 @@ object SharedPackageDefs{ if (ctx.verboseFailures) { ctx.failureTerminalAggregate = startTerminals ctx.failureGroupAggregate = Msgs.empty - ctx.setMsg(startPos, () => "!" + msg.render) + ctx.reportParseMsg(startPos, () => "!" + msg.render, Msgs.empty) } res.cut = startCut res diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 46550c35..c0643bd6 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) res } } else { @@ -153,7 +153,7 @@ object JsonnetWhitespace{ ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) res } } else { @@ -205,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.aggregateTerminal(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) res } } else { diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index b3728a00..4f9f8b43 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -6,14 +6,18 @@ import scala.annotation.{switch, tailrec} import scala.collection.mutable.ArrayBuffer object Util { - def parenthize(fs: Seq[Lazy[String]]) = fs.reverseIterator.map(_()).toSeq.distinct match{ + def parenthize(fs: Lazy[List[String]]) = fs().reverseIterator.toSeq.distinct match{ case Seq(x) => x case xs => xs.mkString("(", " | ", ")") } - def joinBinOp(lhs: Msgs, rhs: Msgs) = - if (lhs.value.isEmpty) rhs - else if (rhs.value.isEmpty) lhs - else Msgs(new Lazy(() => lhs.render + " ~ " + rhs.render) :: Nil) + def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = { + () => + if (lhs.value().isEmpty) rhs.render + else if (rhs.value().isEmpty) lhs.render + else lhs.render + " ~ " + rhs.render + } + + def consumeWhitespace[V](whitespace: fastparse.Whitespace, ctx: ParsingRun[Any]) = { val oldCapturing = ctx.noDropBuffer // completely disallow dropBuffer @@ -100,15 +104,15 @@ object Util { } - def aggregateMsgPostSep[V](startIndex: Int, + def reportParseMsgPostSep[V](startIndex: Int, min: Int, ctx: ParsingRun[Any], parsedMsg: Msgs, lastAgg: Msgs) = { - aggregateMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) + reportParseMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) } - def aggregateMsgInRep[V](startIndex: Int, + def reportParseMsgInRep[V](startIndex: Int, min: Int, ctx: ParsingRun[Any], sepMsg: Msgs, @@ -123,7 +127,7 @@ object Util { if (sepMsg == null || precut) ctx.failureGroupAggregate else Util.joinBinOp(sepMsg, parsedMsg) - ctx.aggregateMsg( + ctx.reportParseMsg( startIndex, () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), if (lastAgg == null) newAgg @@ -179,14 +183,18 @@ final class CompactTrieNode(source: TrieNode){ val word = source.word } object Msgs{ - val empty = Msgs(Nil) + val empty = Msgs(new Lazy(() => Nil)) implicit def fromFunction(msgToSet: () => String): Msgs = { - Msgs(new Lazy(msgToSet) :: Nil) + Msgs(new Lazy(() => msgToSet() :: Nil)) + } + implicit def fromListFunction(msgsToSet: () => List[String]): Msgs = { + Msgs(new Lazy(msgsToSet)) } } -case class Msgs(value: List[Lazy[String]]){ - def :::(other: Msgs) = Msgs(other.value ::: value) - def ::(other: Lazy[String]) = Msgs(other :: value) + +case class Msgs(value: Lazy[List[String]]){ + def :::(other: Msgs) = Msgs(new Lazy(() => other.value() ::: value())) + def ::(other: Lazy[String]) = Msgs(new Lazy(() => other() :: value())) override def toString = render def render = Util.parenthize(value) } \ No newline at end of file diff --git a/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala b/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala index 994ac779..93598e10 100644 --- a/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala +++ b/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala @@ -45,7 +45,7 @@ object CustomWhitespaceMathTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String) = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace().longAggregateMsg + val actualTrace = failure.trace().longreportParseMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index 05ca749a..af642058 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -50,13 +50,13 @@ object ExampleTests extends TestSuite{ trace.longMsg == """Expected parseA:1:1 / "c":1:1, found "d"""" ) - // aggregateMsg and longAggregateMsg record all parsers + // reportParseMsg and longreportParseMsg record all parsers // failing at the position, "a" | "b" | "c", assert( - trace.aggregateMsg == """Expected (parseEither | "c"):1:1, found "d"""", - trace.longAggregateMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" + trace.reportParseMsg == """Expected (parseEither | "c"):1:1, found "d"""", + trace.longreportParseMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" ) } @@ -102,7 +102,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(_, 6) = parse("aaaaab", either(_)) val f @ Parsed.Failure(_, 5, _) = parse("aaaaae", either(_)) - val trace = f.trace().longAggregateMsg + val trace = f.trace().longreportParseMsg assert( f.toString == """Parsed.Failure(Position 1:6, found "e")""", trace == """Expected either:1:1 / ("a" | "b" | "c" | "d"):1:6, found "e"""" @@ -212,7 +212,7 @@ object ExampleTests extends TestSuite{ val failure = parse("", xml(_)).asInstanceOf[Parsed.Failure] assert( - failure.trace().longAggregateMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" + failure.trace().longreportParseMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" ) } test("flatMapFor"){ @@ -230,7 +230,7 @@ object ExampleTests extends TestSuite{ val failure = parse("", xml(_)).asInstanceOf[Parsed.Failure] assert( - failure.trace().longAggregateMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" + failure.trace().longreportParseMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" ) } test("filter"){ @@ -244,7 +244,7 @@ object ExampleTests extends TestSuite{ def letter[$: P] = CharIn("A-Z") def twice[T, $: P](p: => P[T]) = p ~ p def errorMessage[T](p: P[_] => P[T], str: String) = - parse(str, p).asInstanceOf[Parsed.Failure].trace().longAggregateMsg + parse(str, p).asInstanceOf[Parsed.Failure].trace().longreportParseMsg // Portuguese number plate format since 2006 def numberPlate[$: P] = P(twice(digit) ~ "-" ~ twice(letter) ~ "-" ~ twice(digit)) @@ -311,7 +311,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success("abcd", _) = parse("val abcd", nocut(_)) val failure = parse("val 1234", nocut(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 0, trace == """Expected nocut:1:1 / ("val " ~ alpha.rep(1) | "def "):1:1, found "val 1234"""" @@ -325,7 +325,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success("abcd", _) = parse("val abcd", nocut(_)) val failure = parse("val 1234", nocut(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 4, trace == """Expected nocut:1:1 / alpha:1:5 / [a-z]:1:5, found "1234"""" @@ -341,7 +341,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("abcd", "efg"), _) = parse("val abcd; val efg;", stmts(_)) val failure = parse("val abcd; val ", stmts(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 10, trace == """Expected stmts:1:1 / (" " | stmt | end-of-input):1:11, found "val """" @@ -357,7 +357,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("abcd", "efg"), _) = parse("val abcd; val efg;", stmts(_)) val failure = parse("val abcd; val ", stmts(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 14, trace == """Expected stmts:1:1 / stmt:1:11 / alpha:1:15 / [a-z]:1:15, found """"" @@ -371,7 +371,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("1", "23"), _) = parse("(1,23)", tuple(_)) val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 2, trace == """Expected tuple:1:1 / ([0-9] | "," ~ digits | ")"):1:3, found ",)"""" @@ -386,7 +386,7 @@ object ExampleTests extends TestSuite{ val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] val index = failure.index - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( index == 3, trace == """Expected tuple:1:1 / digits:1:4 / [0-9]:1:4, found ")"""" @@ -400,7 +400,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("1", "23"), _) = parse("(1,23)", tuple(_)) val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longAggregateMsg + val trace = failure.trace().longreportParseMsg assert( failure.index == 3, trace == """Expected tuple:1:1 / digits:1:4 / [0-9]:1:4, found ")"""" diff --git a/fastparse/test/src/fastparse/IndentationTests.scala b/fastparse/test/src/fastparse/IndentationTests.scala index 99afd872..5f92fa65 100644 --- a/fastparse/test/src/fastparse/IndentationTests.scala +++ b/fastparse/test/src/fastparse/IndentationTests.scala @@ -126,7 +126,7 @@ object IndentationTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String): Unit = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace(enableLogging = true).longAggregateMsg + val actualTrace = failure.trace(enableLogging = true).longreportParseMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/fastparse/test/src/fastparse/Main.scala b/fastparse/test/src/fastparse/Main.scala index 8365cad9..c47b0323 100644 --- a/fastparse/test/src/fastparse/Main.scala +++ b/fastparse/test/src/fastparse/Main.scala @@ -7,6 +7,6 @@ object Main { def hello[$: P] = P( "hello" ) def combined[$: P] = P( (iam | hello).? ~ ("cow" | "world") ) val Parsed.Failure(_, _, extra) = parse("lol", combined(_)) - println(extra.trace().longAggregateMsg) + println(extra.trace().longreportParseMsg) } } diff --git a/fastparse/test/src/fastparse/MathTests.scala b/fastparse/test/src/fastparse/MathTests.scala index d90d12e4..5bccd209 100644 --- a/fastparse/test/src/fastparse/MathTests.scala +++ b/fastparse/test/src/fastparse/MathTests.scala @@ -33,7 +33,7 @@ object MathTests extends TestSuite{ val Parsed.Success(15, _) = parse("(1+1*2)+3*4", expr(_)) val Parsed.Success(21, _) = parse("((1+1*2)+(3*4*5))/3", expr(_)) val Parsed.Failure(expected, failIndex, extra) = parse("1+1*", expr(_)) - val longAggMsg = extra.trace().longAggregateMsg + val longAggMsg = extra.trace().longreportParseMsg assert( failIndex == 4, longAggMsg == @@ -50,7 +50,7 @@ object MathTests extends TestSuite{ val index = failure.index assert( - expectedTrace.trim == trace.longAggregateMsg.trim, + expectedTrace.trim == trace.longreportParseMsg.trim, expectedTerminalTrace.trim == trace.longTerminalsMsg.trim, expectedShortTrace.trim == failure.msg ) diff --git a/fastparse/test/src/fastparse/WhitespaceMathTests.scala b/fastparse/test/src/fastparse/WhitespaceMathTests.scala index b8617152..9f6b7ff3 100644 --- a/fastparse/test/src/fastparse/WhitespaceMathTests.scala +++ b/fastparse/test/src/fastparse/WhitespaceMathTests.scala @@ -42,7 +42,7 @@ object WhitespaceMathTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String) = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace().longAggregateMsg + val actualTrace = failure.trace().longreportParseMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/scalaparse/test/src/scalaparse/TestUtil.scala b/scalaparse/test/src/scalaparse/TestUtil.scala index 7fdc7349..20d56dc4 100644 --- a/scalaparse/test/src/scalaparse/TestUtil.scala +++ b/scalaparse/test/src/scalaparse/TestUtil.scala @@ -25,18 +25,29 @@ object TestUtil { val parsedTerminals = trace.terminalAggregateString val parsedAggregate = trace.groupAggregateString val parsedFound = input.slice(f.index, f.index + 10) - val stack = trace.longAggregateMsg + val stack = trace.longreportParseMsg - assert( - { implicitly(input) + assert({ + implicitly(input) + implicitly(stack) + implicitly(index) + implicitly(parsedFound) + (aggregate == null || aggregate.trim == parsedAggregate.trim) + }) + assert({ + implicitly(input) + implicitly(stack) + implicitly(index) + implicitly(parsedFound) + (terminals == null || terminals.trim == parsedTerminals.trim) + }) + assert({ + implicitly(input) implicitly(stack) implicitly(index) implicitly(parsedFound) - (aggregate == null || aggregate.trim == parsedAggregate.trim) && - (terminals == null || terminals.trim == parsedTerminals.trim) && parsedFound.startsWith(found) - } - ) + }) line.value case _: Parsed.Success[_] => diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 01e84569..273c31ad 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -43,7 +43,7 @@ object FailureTests extends TestSuite{ |} """.stripMargin, aggregate = """(NamedType | Refinement)""", - terminals = """(chars-while(IdCharacter, 1) | [_] | [ \t] | "/*" | "//" | "(" | "-" | "." | [0-9] | "0x" | "true" | "false" | "`" | char-pred(UpperChar) | char-pred(LowerChar) | var-id | chars-while(OpCharNotSlash, 1) | "/" | operator | plain-id | id | filter | "\"\"\"" | "\"" | "'" | "null" | "this" | "super" | "_" | "{")""", + terminals = """(chars-while(IdCharacter, 1) | [_] | [ \t] | "/*" | "//" | "\n" | "\r\n" | "(" | "-" | "." | [0-9] | "0x" | "true" | "false" | "`" | char-pred(UpperChar) | char-pred(LowerChar) | var-id | chars-while(OpCharNotSlash, 1) | "/" | operator | plain-id | id | filter | "\"\"\"" | "\"" | "'" | "null" | "this" | "super" | "_" | "{")""", found = ")" ) test - checkNeg( From 1520e91a45d1d75131888ce45a87a3faab91e4e0 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 17:53:32 +0800 Subject: [PATCH 10/49] prettify --- fastparse/src/fastparse/ParsingRun.scala | 32 +++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 204d51f1..68915062 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -183,12 +183,36 @@ final class ParsingRun[+T](val input: ParserInput, /** * Called by non-terminal parsers after completion, success or failure + * + * This needs to be called for both successful and failed parsers, as we need + * to record the msg of a successful parse in case it forms part of a larger + * failed parse later. + * + * For example: + * + * - Using "a" ~ ("b" ~ "c" | "d") to parse "abe" + * - We report that the the parser ("b" ~ "c" | "d") failed at index 1 + * - That msg contains the msg of the parse "b" even though it was successful */ + def reportParseMsg(startIndex: Int, + msgToSet: Msgs, + msgToAggregate: Msgs): Unit = { + + reportParseMsg(startIndex, msgToSet, msgToAggregate, false) + } + def reportParseMsg(startIndex: Int, msgToSet: Msgs, msgToAggregate: Msgs, - forceAggregate: Boolean = false, - setShortMsg: Boolean = true): Unit = { + forceAggregate: Boolean): Unit = { + reportParseMsg0(startIndex, msgToSet, msgToAggregate, forceAggregate, true) + } + + def reportParseMsg0(startIndex: Int, + msgToSet: Msgs, + msgToAggregate: Msgs, + forceAggregate: Boolean, + setShortMsg: Boolean): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet @@ -210,8 +234,10 @@ final class ParsingRun[+T](val input: ParserInput, */ def reportTerminalParseMsg(startIndex: Int, msgToSet: Msgs): Unit = { + // We only care about terminal parsers which failed exactly at the traceIndex if (!isSuccess && index == traceIndex) failureTerminalAggregate :::= msgToSet - reportParseMsg( + + reportParseMsg0( startIndex, msgToSet, Msgs.empty, From afc8454d7e5367b569de849a249fbcb19a0ede2b Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 17:55:53 +0800 Subject: [PATCH 11/49] prettify --- cssparse/test/src-jvm/cssparse/TestUtil.scala | 2 +- fastparse/src/fastparse/Parsed.scala | 4 +-- .../fastparse/CustomWhitespaceMathTests.scala | 2 +- .../test/src/fastparse/ExampleTests.scala | 26 +++++++++---------- .../test/src/fastparse/IndentationTests.scala | 2 +- fastparse/test/src/fastparse/Main.scala | 2 +- fastparse/test/src/fastparse/MathTests.scala | 4 +-- .../src/fastparse/WhitespaceMathTests.scala | 2 +- scalaparse/test/src/scalaparse/TestUtil.scala | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cssparse/test/src-jvm/cssparse/TestUtil.scala b/cssparse/test/src-jvm/cssparse/TestUtil.scala index 05bec257..fdd3bb3d 100644 --- a/cssparse/test/src-jvm/cssparse/TestUtil.scala +++ b/cssparse/test/src-jvm/cssparse/TestUtil.scala @@ -9,7 +9,7 @@ object TestUtil { def checkParsed(input: String, res: Parsed[Ast.RuleList]) = { res match { case f: Parsed.Failure => - throw new Exception(tag + "\n" + input + "\n" + f.trace().longreportParseMsg) + throw new Exception(tag + "\n" + input + "\n" + f.trace().longAggregateMsg) case s: Parsed.Success[Ast.RuleList] => val inputLength = input.length val index = s.index diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index f54a795f..4abd631f 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -75,7 +75,7 @@ object Parsed{ Failure.formatMsg(extra.input, extra.stack ++ List(label -> index), index) } else throw new Exception( "`.longMsg` requires the parser to be run with `verboseFailures = true`, " + - "or to be called via `.trace().longMsg` or `.trace().longreportParseMsg`" + "or to be called via `.trace().longMsg` or `.trace().longAggregateMsg`" ) } @@ -218,6 +218,6 @@ object Parsed{ /** * A version of [[reportParseMsg]] that includes the parse stack */ - def longreportParseMsg = Failure.formatMsg(input, stack ++ Seq(groupAggregateString -> index), index) + def longAggregateMsg = Failure.formatMsg(input, stack ++ Seq(groupAggregateString -> index), index) } } diff --git a/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala b/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala index 93598e10..994ac779 100644 --- a/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala +++ b/fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala @@ -45,7 +45,7 @@ object CustomWhitespaceMathTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String) = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace().longreportParseMsg + val actualTrace = failure.trace().longAggregateMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index af642058..765d7253 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -50,13 +50,13 @@ object ExampleTests extends TestSuite{ trace.longMsg == """Expected parseA:1:1 / "c":1:1, found "d"""" ) - // reportParseMsg and longreportParseMsg record all parsers + // reportParseMsg and longAggregateMsg record all parsers // failing at the position, "a" | "b" | "c", assert( trace.reportParseMsg == """Expected (parseEither | "c"):1:1, found "d"""", - trace.longreportParseMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" + trace.longAggregateMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" ) } @@ -102,7 +102,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(_, 6) = parse("aaaaab", either(_)) val f @ Parsed.Failure(_, 5, _) = parse("aaaaae", either(_)) - val trace = f.trace().longreportParseMsg + val trace = f.trace().longAggregateMsg assert( f.toString == """Parsed.Failure(Position 1:6, found "e")""", trace == """Expected either:1:1 / ("a" | "b" | "c" | "d"):1:6, found "e"""" @@ -212,7 +212,7 @@ object ExampleTests extends TestSuite{ val failure = parse("", xml(_)).asInstanceOf[Parsed.Failure] assert( - failure.trace().longreportParseMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" + failure.trace().longAggregateMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" ) } test("flatMapFor"){ @@ -230,7 +230,7 @@ object ExampleTests extends TestSuite{ val failure = parse("", xml(_)).asInstanceOf[Parsed.Failure] assert( - failure.trace().longreportParseMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" + failure.trace().longAggregateMsg == """Expected xml:1:1 / rightTag:1:8 / "abcde":1:10, found "edcba>"""" ) } test("filter"){ @@ -244,7 +244,7 @@ object ExampleTests extends TestSuite{ def letter[$: P] = CharIn("A-Z") def twice[T, $: P](p: => P[T]) = p ~ p def errorMessage[T](p: P[_] => P[T], str: String) = - parse(str, p).asInstanceOf[Parsed.Failure].trace().longreportParseMsg + parse(str, p).asInstanceOf[Parsed.Failure].trace().longAggregateMsg // Portuguese number plate format since 2006 def numberPlate[$: P] = P(twice(digit) ~ "-" ~ twice(letter) ~ "-" ~ twice(digit)) @@ -311,7 +311,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success("abcd", _) = parse("val abcd", nocut(_)) val failure = parse("val 1234", nocut(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 0, trace == """Expected nocut:1:1 / ("val " ~ alpha.rep(1) | "def "):1:1, found "val 1234"""" @@ -325,7 +325,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success("abcd", _) = parse("val abcd", nocut(_)) val failure = parse("val 1234", nocut(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 4, trace == """Expected nocut:1:1 / alpha:1:5 / [a-z]:1:5, found "1234"""" @@ -341,7 +341,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("abcd", "efg"), _) = parse("val abcd; val efg;", stmts(_)) val failure = parse("val abcd; val ", stmts(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 10, trace == """Expected stmts:1:1 / (" " | stmt | end-of-input):1:11, found "val """" @@ -357,7 +357,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("abcd", "efg"), _) = parse("val abcd; val efg;", stmts(_)) val failure = parse("val abcd; val ", stmts(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 14, trace == """Expected stmts:1:1 / stmt:1:11 / alpha:1:15 / [a-z]:1:15, found """"" @@ -371,7 +371,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("1", "23"), _) = parse("(1,23)", tuple(_)) val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 2, trace == """Expected tuple:1:1 / ([0-9] | "," ~ digits | ")"):1:3, found ",)"""" @@ -386,7 +386,7 @@ object ExampleTests extends TestSuite{ val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] val index = failure.index - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( index == 3, trace == """Expected tuple:1:1 / digits:1:4 / [0-9]:1:4, found ")"""" @@ -400,7 +400,7 @@ object ExampleTests extends TestSuite{ val Parsed.Success(Seq("1", "23"), _) = parse("(1,23)", tuple(_)) val failure = parse("(1,)", tuple(_)).asInstanceOf[Parsed.Failure] - val trace = failure.trace().longreportParseMsg + val trace = failure.trace().longAggregateMsg assert( failure.index == 3, trace == """Expected tuple:1:1 / digits:1:4 / [0-9]:1:4, found ")"""" diff --git a/fastparse/test/src/fastparse/IndentationTests.scala b/fastparse/test/src/fastparse/IndentationTests.scala index 5f92fa65..99afd872 100644 --- a/fastparse/test/src/fastparse/IndentationTests.scala +++ b/fastparse/test/src/fastparse/IndentationTests.scala @@ -126,7 +126,7 @@ object IndentationTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String): Unit = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace(enableLogging = true).longreportParseMsg + val actualTrace = failure.trace(enableLogging = true).longAggregateMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/fastparse/test/src/fastparse/Main.scala b/fastparse/test/src/fastparse/Main.scala index c47b0323..8365cad9 100644 --- a/fastparse/test/src/fastparse/Main.scala +++ b/fastparse/test/src/fastparse/Main.scala @@ -7,6 +7,6 @@ object Main { def hello[$: P] = P( "hello" ) def combined[$: P] = P( (iam | hello).? ~ ("cow" | "world") ) val Parsed.Failure(_, _, extra) = parse("lol", combined(_)) - println(extra.trace().longreportParseMsg) + println(extra.trace().longAggregateMsg) } } diff --git a/fastparse/test/src/fastparse/MathTests.scala b/fastparse/test/src/fastparse/MathTests.scala index 5bccd209..d90d12e4 100644 --- a/fastparse/test/src/fastparse/MathTests.scala +++ b/fastparse/test/src/fastparse/MathTests.scala @@ -33,7 +33,7 @@ object MathTests extends TestSuite{ val Parsed.Success(15, _) = parse("(1+1*2)+3*4", expr(_)) val Parsed.Success(21, _) = parse("((1+1*2)+(3*4*5))/3", expr(_)) val Parsed.Failure(expected, failIndex, extra) = parse("1+1*", expr(_)) - val longAggMsg = extra.trace().longreportParseMsg + val longAggMsg = extra.trace().longAggregateMsg assert( failIndex == 4, longAggMsg == @@ -50,7 +50,7 @@ object MathTests extends TestSuite{ val index = failure.index assert( - expectedTrace.trim == trace.longreportParseMsg.trim, + expectedTrace.trim == trace.longAggregateMsg.trim, expectedTerminalTrace.trim == trace.longTerminalsMsg.trim, expectedShortTrace.trim == failure.msg ) diff --git a/fastparse/test/src/fastparse/WhitespaceMathTests.scala b/fastparse/test/src/fastparse/WhitespaceMathTests.scala index 9f6b7ff3..b8617152 100644 --- a/fastparse/test/src/fastparse/WhitespaceMathTests.scala +++ b/fastparse/test/src/fastparse/WhitespaceMathTests.scala @@ -42,7 +42,7 @@ object WhitespaceMathTests extends TestSuite{ test("fail"){ def check(input: String, expectedTrace: String) = { val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] - val actualTrace = failure.trace().longreportParseMsg + val actualTrace = failure.trace().longAggregateMsg assert(expectedTrace.trim == actualTrace.trim) } test - check( diff --git a/scalaparse/test/src/scalaparse/TestUtil.scala b/scalaparse/test/src/scalaparse/TestUtil.scala index 20d56dc4..ae2fd9dc 100644 --- a/scalaparse/test/src/scalaparse/TestUtil.scala +++ b/scalaparse/test/src/scalaparse/TestUtil.scala @@ -25,7 +25,7 @@ object TestUtil { val parsedTerminals = trace.terminalAggregateString val parsedAggregate = trace.groupAggregateString val parsedFound = input.slice(f.index, f.index + 10) - val stack = trace.longreportParseMsg + val stack = trace.longAggregateMsg assert({ implicitly(input) From 8ff1dd3ccfb8bb0a2dc1d7eccbcf1bb16487972b Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 18:01:06 +0800 Subject: [PATCH 12/49] prettify --- fastparse/src/fastparse/ParsingRun.scala | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 68915062..7ca87da7 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -160,11 +160,15 @@ final class ParsingRun[+T](val input: ParserInput, // `failureGroupAggregate` would be wasteful. // // These is an edge case where there is no given failure that occurs exactly at - // `traceIndex` e.g. parsing "ax" with P( ("a" ~ "b") ~ "c" | "a" ~/ "d" ), the - // final failure `index` and thus `traceIndex` is at offset 1, and we would like - // to receive the aggregation ("b" | "d"). But ("a" ~ "b") - // passes from offsets 0-2, "c" fails at offset 2 and ("a" ~ "b") ~ "c" fails - // from offset 0-2. In such a case, we truncate the `shortParserMsg` at + // `traceIndex` e.g. + // + // - Parsing "ax" with P( ("a" ~ "b") ~ "c" | "a" ~/ "d" ) + // - The final failure `index` and thus `traceIndex` is at offset 1 + // - We would like to receive the aggregation ("b" | "d") + // - But ("a" ~ "b") passes from offsets 0-2, "c" fails at offset 2 and ("a" ~ "b") ~ "c" fails + // from offset 0-2. + // + // In such a case, we truncate the `shortParserMsg` at // `traceIndex` to only include the portion we're interested in (which directly // follows the failure). This then gets aggregated nicely to form the error // message from-point-of-failure. @@ -174,12 +178,14 @@ final class ParsingRun[+T](val input: ParserInput, // val inner = P( "a" ~ "b" ) // P( inner ~ "c" | "a" ~/ "d" ) // - // Here, we find that the `inner` parser starts before the `traceIndex` and - // fails at `traceIndex`, but we want our aggregation to continue being - // ("b" | "d"), rather than (inner | "d"). Thus, for opaque compound parsers - // like `inner` which do not expose their internals, we use the `forceAggregate` - // to force it to expose it's internals when it's range covers the `traceIndex` - // but it isn't an exact match + // - Here, we find that the `inner` parser starts before the `traceIndex` and + // fails at `traceIndex`, + // - But we want our aggregation to continue being ("b" | "d"), rather than + // (inner | "d"). + // + // Thus, for opaque compound parsers like `inner` which do not expose their + // internals, we use `forceAggregate` to force it to expose it's internals + // when it's range covers the `traceIndex` but it isn't an exact match /** * Called by non-terminal parsers after completion, success or failure From 2454f7a152b7d12a91b46593e02367d73200532d Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 22:13:50 +0800 Subject: [PATCH 13/49] renaming --- .../src-2/fastparse/internal/MacroImpls.scala | 14 +++---- .../fastparse/internal/MacroRepImpls.scala | 2 +- .../src-2/fastparse/internal/RepImpls.scala | 8 ++-- .../fastparse/internal/MacroInlineImpls.scala | 14 +++---- .../fastparse/internal/MacroRepImpls.scala | 2 +- fastparse/src/fastparse/Parsed.scala | 4 +- fastparse/src/fastparse/ParsingRun.scala | 42 +++++++++---------- .../src/fastparse/SharedPackageDefs.scala | 16 +++---- fastparse/src/fastparse/internal/Util.scala | 2 +- .../test/src/fastparse/FailureTests.scala | 2 +- 10 files changed, 53 insertions(+), 53 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 619da612..3b740f7c 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -55,7 +55,7 @@ object MacroImpls { ctx0.reportParseMsg( startIndex, () => name.splice.value, - ctx0.failureGroupAggregate, + ctx0.failureGroups, startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess){ @@ -232,7 +232,7 @@ object MacroImpls { lhs0.splice val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureGroupAggregate + val lhsAggregate = ctx5.failureGroups if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -251,7 +251,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -466,7 +466,7 @@ object MacroImpls { else { val $preRhsIndex = $ctx1.index $rhs - val $rhsAggregate = $ctx1.failureGroupAggregate + val $rhsAggregate = $ctx1.failureGroups val $rhsMsg = $ctx1.shortParserMsg val $res = if (!$ctx1.isSuccess) { @@ -492,7 +492,7 @@ object MacroImpls { $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, - // We override the failureGroupAggregate to avoid building an `a ~ b` + // We override the failureGroups to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -522,7 +522,7 @@ object MacroImpls { if (!$ctx1.isSuccess) $ctx1 else { val $postLhsIndex = $ctx1.index - val $lhsAggregate = $ctx1.failureGroupAggregate + val $lhsAggregate = $ctx1.failureGroups val $lhsMsg = $ctx1.shortParserMsg $setCut @@ -693,7 +693,7 @@ object MacroImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg - val agg = ctx1.failureGroupAggregate + val agg = ctx1.failureGroups if (!postSuccess){ ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) } diff --git a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala index 03fea383..bba8a5fb 100644 --- a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala @@ -75,7 +75,7 @@ object MacroRepImpls{ ${c.prefix}.parse0() val $parsedMsg = $ctx1.shortParserMsg - val $parsedAgg = $ctx1.failureGroupAggregate + val $parsedAgg = $ctx1.failureGroups $originalCut |= $ctx1.cut if (!$ctx1.isSuccess) { val res = diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index 7e3835fb..91500c36 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -34,7 +34,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val verboseFailures = ctx.verboseFailures parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroupAggregate + val parsedAgg = ctx.failureGroups val postCut = ctx.cut if (!ctx.isSuccess) { val res = @@ -95,7 +95,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroupAggregate + val parsedAgg = ctx.failureGroups val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -156,7 +156,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ else { parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroupAggregate + val parsedAgg = ctx.failureGroups val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -224,7 +224,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroupAggregate + val parsedAgg = ctx.failureGroups val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess){ diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 1405bbb9..e1ceb9f8 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -106,7 +106,7 @@ object MacroInlineImpls { ctx0.reportParseMsg( startIndex, () => name.value, - ctx0.failureGroupAggregate, + ctx0.failureGroups, startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess) { @@ -149,7 +149,7 @@ object MacroInlineImpls { if (!ctx1.isSuccess) ctx1 else { val postLhsIndex = ctx1.index - val lhsAggregate = ctx1.failureGroupAggregate + val lhsAggregate = ctx1.failureGroups val lhsMsg = ctx1.shortParserMsg ${ setCut('{ ctx1 }) } @@ -163,7 +163,7 @@ object MacroInlineImpls { else { val preRhsIndex = ctx1.index $rhs - val rhsAggregate = ctx1.failureGroupAggregate + val rhsAggregate = ctx1.failureGroups val rhsMsg = ctx1.shortParserMsg val res = if (!ctx1.isSuccess) { @@ -192,7 +192,7 @@ object MacroInlineImpls { preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, - // We override the failureGroupAggregate to avoid building an `a ~ b` + // We override the failureGroups to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -245,7 +245,7 @@ object MacroInlineImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg - val agg = ctx1.failureGroupAggregate + val agg = ctx1.failureGroups if (!postSuccess) { ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) } @@ -301,7 +301,7 @@ object MacroInlineImpls { lhs0 val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureGroupAggregate + val lhsAggregate = ctx5.failureGroups if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -320,7 +320,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) - ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroupAggregate ::: lhsAggregate) + ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 246f7dd9..b72959ed 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -98,7 +98,7 @@ object MacroRepImpls { else { $parse0 val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroupAggregate + val parsedAgg = ctx.failureGroups val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index 4abd631f..dfe5ee22 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -148,8 +148,8 @@ object Parsed{ def fromParsingRun[T](p: ParsingRun[T]) = { assert(!p.isSuccess) TracedFailure( - p.failureTerminalAggregate, - p.lastFailureMsg ::: p.failureGroupAggregate, + p.failureTerminals, + p.lastFailureMsg ::: p.failureGroups, Parsed.fromParsingRun(p).asInstanceOf[Failure] ) } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 7ca87da7..44e16d3e 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -35,10 +35,10 @@ import fastparse.internal.{Instrument, Lazy, Msgs, Util} * it with tracing enabled. * @param traceIndex The index we wish to trace if tracing is enabled, else * -1. Used to find failure messages to aggregate into - * `failureTerminalAggregate` + * `failureTerminals` * @param instrument Callbacks that can be injected before/after every * `P(...)` parser. - * @param failureTerminalAggregate When tracing is enabled, this collects up all the + * @param failureTerminals When tracing is enabled, this collects up all the * upper-most failures that happen at [[traceIndex]] * (in [[Lazy]] wrappers) so they can be shown to the * user at end-of-parse as suggestions for what could @@ -108,8 +108,8 @@ final class ParsingRun[+T](val input: ParserInput, val traceIndex: Int, val instrument: Instrument, // Mutable vars below: - var failureTerminalAggregate: Msgs, - var failureGroupAggregate: Msgs, + var failureTerminals: Msgs, + var failureGroups: Msgs, var shortParserMsg: Msgs, var lastFailureMsg: Msgs, var failureStack: List[(String, Int)], @@ -125,39 +125,39 @@ final class ParsingRun[+T](val input: ParserInput, // HOW ERROR AGGREGATION WORKS: // // Fastparse provides two levels of error aggregation that get enabled when - // calling `.trace()`: `failureTerminalAggregate`, and `failureGroupAggregate`: + // calling `.trace()`: `failureTerminals`, and `failureGroups`: // - // - `failureTerminalAggregate` lists all low-level terminal parsers which are + // - `failureTerminals` lists all low-level terminal parsers which are // tried at the given `traceIndex`. This is useful to answer the question // "what can I put at the error position to make my parse continue" // - // - `failureGroupAggregate` lists all high-level parsers which are tried at + // - `failureGroups` lists all high-level parsers which are tried at // the given `traceIndex`. This is useful to answer the question "What was // the parser trying to do when it failed" // - // The implementation of `failureTerminalAggregate` is straightforward: we + // The implementation of `failureTerminals` is straightforward: we // simply call `reportTerminalParseMsg` in every terminal parser, which collects // all the messages in a big list and returns it. The implementation of - // `failureGroupAggregate` is more interesting, since we need to figure out + // `failureGroups` is more interesting, since we need to figure out // what are the "high level" parsers that we need to list. We use the // following algorithm: // // - When a parse which started at the given `traceIndex` fails without a cut - // - Over-write `failureGroupAggregate` with it's `shortParserMsg` + // - Over-write `failureGroups` with it's `shortParserMsg` // // - Otherwise: - // - If we are a terminal parser, we set our `failureGroupAggregate` to Nil - // - If we are a compound parser, we simply sum up the `failureGroupAggregate` + // - If we are a terminal parser, we set our `failureGroups` to Nil + // - If we are a compound parser, we simply sum up the `failureGroups` // of all our constituent parts // // The point of this heuristic is to provide the highest-level parsers which // failed at the `traceIndex`, but are not already part of the `failureStack`. // non-highest-level parsers do successfully write their message to - // `failureGroupAggregate`, but they are subsequently over-written by the higher + // `failureGroups`, but they are subsequently over-written by the higher // level parsers, until it reaches the point where `cut == true`, indicating // that any further higher-level parsers will be in `failureStack` and using // their message to stomp over the existing parse-failure-messages in - // `failureGroupAggregate` would be wasteful. + // `failureGroups` would be wasteful. // // These is an edge case where there is no given failure that occurs exactly at // `traceIndex` e.g. @@ -215,10 +215,10 @@ final class ParsingRun[+T](val input: ParserInput, } def reportParseMsg0(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs, - forceAggregate: Boolean, - setShortMsg: Boolean): Unit = { + msgToSet: Msgs, + msgToAggregate: Msgs, + forceAggregate: Boolean, + setShortMsg: Boolean): Unit = { if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet @@ -227,7 +227,7 @@ final class ParsingRun[+T](val input: ParserInput, // There are two cases when aggregating: either we stomp over the entire // existing aggregation with `msgToSet`, or we preserve it (with possible // additions) with `msgToAggregate`. - failureGroupAggregate = + failureGroups = if (checkAggregate(startIndex) && !forceAggregate) shortParserMsg else msgToAggregate } @@ -235,13 +235,13 @@ final class ParsingRun[+T](val input: ParserInput, /** * Called by any terminal parser; these are the smallest parsers that a user * may care about, e.g. individual strings or characters, and will be stored - * in the `failureTerminalAggregate` in case a user wants to know what could + * in the `failureTerminals` in case a user wants to know what could * have been placed at the failure point to let the parse progress */ def reportTerminalParseMsg(startIndex: Int, msgToSet: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex - if (!isSuccess && index == traceIndex) failureTerminalAggregate :::= msgToSet + if (!isSuccess && index == traceIndex) failureTerminals :::= msgToSet reportParseMsg0( startIndex, diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 634b1e22..4955152d 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -54,8 +54,8 @@ trait SharedPackageDefs { originalParser = parser, traceIndex = traceIndex, instrument = instrument, - failureTerminalAggregate = Msgs.empty, - failureGroupAggregate = Msgs.empty, + failureTerminals = Msgs.empty, + failureGroups = Msgs.empty, shortParserMsg = Msgs.empty, lastFailureMsg = null, failureStack = List.empty, @@ -105,7 +105,7 @@ trait SharedPackageDefs { if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { - ctx.failureGroupAggregate = Msgs.empty + ctx.failureGroups = Msgs.empty ctx.reportTerminalParseMsg(startPos, () => msg match{ case Seq(x) => s"&(${msg.render})" @@ -144,7 +144,7 @@ trait SharedPackageDefs { /** * Wraps a parser and ensures that none of the parsers within it leave - * failure traces in failureTerminalAggregate, though unlike [[ByNameOps.opaque]] + * failure traces in failureTerminals, though unlike [[ByNameOps.opaque]] * if there is a failure *within* the wrapped parser the failure's location * and error message will still be shown * @@ -156,7 +156,7 @@ trait SharedPackageDefs { val res = p if (ctx.verboseFailures) { - ctx.failureGroupAggregate = Msgs.empty + ctx.failureGroups = Msgs.empty ctx.shortParserMsg = Msgs.empty } res @@ -272,7 +272,7 @@ object SharedPackageDefs{ val startCut = ctx.cut val oldNoCut = ctx.noDropBuffer ctx.noDropBuffer = true - val startTerminals = ctx.failureTerminalAggregate + val startTerminals = ctx.failureTerminals parse0() ctx.noDropBuffer = oldNoCut val msg = ctx.shortParserMsg @@ -282,8 +282,8 @@ object SharedPackageDefs{ else ctx.freshSuccessUnit(startPos) if (ctx.verboseFailures) { - ctx.failureTerminalAggregate = startTerminals - ctx.failureGroupAggregate = Msgs.empty + ctx.failureTerminals = startTerminals + ctx.failureGroups = Msgs.empty ctx.reportParseMsg(startPos, () => "!" + msg.render, Msgs.empty) } res.cut = startCut diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 4f9f8b43..08daf7f4 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -124,7 +124,7 @@ object Util { // we backtrack past the sep on failure) as well as the failure // aggregate of the previous rep, which we could have continued val newAgg = - if (sepMsg == null || precut) ctx.failureGroupAggregate + if (sepMsg == null || precut) ctx.failureGroups else Util.joinBinOp(sepMsg, parsedMsg) ctx.reportParseMsg( diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 50cb6086..199e10c4 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -364,7 +364,7 @@ object FailureTests extends TestSuite{ import NoWhitespace._ // In the case where one branch fails further in than `traceIndex`, we // collect the partial aggregation from that branch in the - // `failureGroupAggregate` but ignore that branch's downstream failure in + // `failureGroups` but ignore that branch's downstream failure in // `failureTerminalsAggregate` def check(parser: P[_] => P[_]) = checkOffset( From 0b6f403ac119c493ab2ce48bac20cef6559bf397 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 22:28:32 +0800 Subject: [PATCH 14/49] . --- .../src-2/fastparse/internal/MacroImpls.scala | 6 +-- .../fastparse/internal/MacroInlineImpls.scala | 7 ++- fastparse/src/fastparse/ParsingRun.scala | 43 +++++++++++++------ .../test/src/fastparse/FailureTests.scala | 10 +++++ 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 3b740f7c..bdf66325 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -55,7 +55,6 @@ object MacroImpls { ctx0.reportParseMsg( startIndex, () => name.splice.value, - ctx0.failureGroups, startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess){ @@ -242,7 +241,7 @@ object MacroImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg, lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg) ctx5.cut = false other.splice @@ -693,9 +692,8 @@ object MacroImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg - val agg = ctx1.failureGroups if (!postSuccess){ - ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) + ctx1.reportParseMsg(startPos, () => msg.render + ".?") } } res diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index e1ceb9f8..c3447728 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -106,7 +106,7 @@ object MacroInlineImpls { ctx0.reportParseMsg( startIndex, () => name.value, - ctx0.failureGroups, + startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess) { @@ -245,9 +245,8 @@ object MacroInlineImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg - val agg = ctx1.failureGroups if (!postSuccess) { - ctx1.reportParseMsg(startPos, () => msg.render + ".?", agg) + ctx1.reportParseMsg(startPos, () => msg.render + ".?") } } res @@ -310,7 +309,7 @@ object MacroInlineImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg, lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg) ctx5.cut = false other diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 44e16d3e..ab25c620 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -199,37 +199,52 @@ final class ParsingRun[+T](val input: ParserInput, * - Using "a" ~ ("b" ~ "c" | "d") to parse "abe" * - We report that the the parser ("b" ~ "c" | "d") failed at index 1 * - That msg contains the msg of the parse "b" even though it was successful + * + * Overloaded to minimize the amount of callsite bytecode, since we do a ton + * of inlining in Fastparse, and large amounts of bytecode inlined in a method + * can cause JVM performance problems (e.g. JIT compilation may get disabled) */ def reportParseMsg(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs): Unit = { + newShortParserMsg: Msgs): Unit = { - reportParseMsg(startIndex, msgToSet, msgToAggregate, false) + reportParseMsg(startIndex, newShortParserMsg, failureGroups) + } + def reportParseMsg(startIndex: Int, + newShortParserMsg: Msgs, + newFailureGroups: Msgs): Unit = { + + reportParseMsg(startIndex, newShortParserMsg, newFailureGroups, false) + } + + def reportParseMsg(startIndex: Int, + newShortParserMsg: Msgs, + forceAggregate: Boolean): Unit = { + reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, true) } def reportParseMsg(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs, + newShortParserMsg: Msgs, + newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, msgToSet, msgToAggregate, forceAggregate, true) + reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) } def reportParseMsg0(startIndex: Int, - msgToSet: Msgs, - msgToAggregate: Msgs, + newShortParserMsg: Msgs, + newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { - if (!isSuccess && lastFailureMsg == null) lastFailureMsg = msgToSet + if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg - shortParserMsg = if (setShortMsg) msgToSet else Msgs.empty + shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty // There are two cases when aggregating: either we stomp over the entire // existing aggregation with `msgToSet`, or we preserve it (with possible // additions) with `msgToAggregate`. failureGroups = if (checkAggregate(startIndex) && !forceAggregate) shortParserMsg - else msgToAggregate + else newFailureGroups } /** @@ -239,13 +254,13 @@ final class ParsingRun[+T](val input: ParserInput, * have been placed at the failure point to let the parse progress */ def reportTerminalParseMsg(startIndex: Int, - msgToSet: Msgs): Unit = { + newShortParserMsg: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex - if (!isSuccess && index == traceIndex) failureTerminals :::= msgToSet + if (!isSuccess && index == traceIndex) failureTerminals :::= newShortParserMsg reportParseMsg0( startIndex, - msgToSet, + newShortParserMsg, Msgs.empty, false, startIndex >= traceIndex diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 199e10c4..e0c4c913 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -126,6 +126,16 @@ object FailureTests extends TestSuite{ assert(trace2.groupAggregateString == """("," ~ parseB | "c")""") f2.index } + test("repTooFew"){ + def parseB[$: P] = P( "a" | "b" ) + def parseA[$: P] = P( parseB.rep(5) ) + val f1 @ Parsed.Failure(_, _, _) = parse("abab", parseA(_)) + + val trace = f1.trace() + + assert(trace.groupAggregateString == """("a" | "b")""") + assert(trace.terminalAggregateString == """("a" | "b")""") + } test("sepCut"){ def parseB[$: P] = P( "a" | "b" | "c" ) From 26df055dc7a715860c395678650bf0a3ab54c9b5 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 22:44:51 +0800 Subject: [PATCH 15/49] . --- fastparse/src-2/fastparse/internal/MacroImpls.scala | 2 +- fastparse/src-3/fastparse/internal/MacroInlineImpls.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index bdf66325..ca1e4c24 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -55,7 +55,7 @@ object MacroImpls { ctx0.reportParseMsg( startIndex, () => name.splice.value, - startIndex < ctx0.traceIndex + forceAggregate = startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess){ ctx0.failureStack = (name.splice.value -> startIndex) :: ctx0.failureStack diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index c3447728..2141bca7 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -106,8 +106,7 @@ object MacroInlineImpls { ctx0.reportParseMsg( startIndex, () => name.value, - - startIndex < ctx0.traceIndex + forceAggregate = startIndex < ctx0.traceIndex ) if (!ctx0.isSuccess) { ctx0.failureStack = (name.value -> startIndex) :: ctx0.failureStack From 8134ac7107438a8d8cfdbff1f7fcfc10596dcd58 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 4 Mar 2023 22:58:48 +0800 Subject: [PATCH 16/49] . --- fastparse/src/fastparse/ParsingRun.scala | 45 ++++++++++++------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index ab25c620..edc8d4fc 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -199,9 +199,9 @@ final class ParsingRun[+T](val input: ParserInput, * - Using "a" ~ ("b" ~ "c" | "d") to parse "abe" * - We report that the the parser ("b" ~ "c" | "d") failed at index 1 * - That msg contains the msg of the parse "b" even though it was successful - * - * Overloaded to minimize the amount of callsite bytecode, since we do a ton - * of inlining in Fastparse, and large amounts of bytecode inlined in a method + * + * Overloaded to minimize the amount of callsite bytecode, since we do a ton + * of inlining in Fastparse, and large amounts of bytecode inlined in a method * can cause JVM performance problems (e.g. JIT compilation may get disabled) */ def reportParseMsg(startIndex: Int, @@ -229,24 +229,6 @@ final class ParsingRun[+T](val input: ParserInput, reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) } - def reportParseMsg0(startIndex: Int, - newShortParserMsg: Msgs, - newFailureGroups: Msgs, - forceAggregate: Boolean, - setShortMsg: Boolean): Unit = { - - if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg - - shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty - - // There are two cases when aggregating: either we stomp over the entire - // existing aggregation with `msgToSet`, or we preserve it (with possible - // additions) with `msgToAggregate`. - failureGroups = - if (checkAggregate(startIndex) && !forceAggregate) shortParserMsg - else newFailureGroups - } - /** * Called by any terminal parser; these are the smallest parsers that a user * may care about, e.g. individual strings or characters, and will be stored @@ -267,10 +249,29 @@ final class ParsingRun[+T](val input: ParserInput, ) } + def reportParseMsg0(startIndex: Int, + newShortParserMsg: Msgs, + newFailureGroups: Msgs, + forceAggregate: Boolean, + setShortMsg: Boolean): Unit = { + + if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg + + shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty + + // There are two cases when aggregating: either we stomp over the entire + // existing `failureGroups` with `newShortParserMsg`, or we preserve it + // (with possible additions) with `newFailureGroups`. + failureGroups = + if (forceAggregate) newFailureGroups + else if (discardNewFailureGroups(startIndex)) shortParserMsg + else newFailureGroups + } + /** * Conditions under which we want to aggregate the given parse */ - def checkAggregate(startIndex: Int) = { + def discardNewFailureGroups(startIndex: Int) = { // We only aggregate if we are not currently past a cut; if we are past a // cut, there is no further backtracking and so the error aggregate that has // occurred will be the final aggregate shown to the user From 9f1669214c8fdf90cc97df5527801e864be88489 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 01:16:49 +0800 Subject: [PATCH 17/49] fix bug with rep separator being included in error msg even when a cut is present to prevent backtracking --- .../src-2/fastparse/internal/RepImpls.scala | 8 ++--- .../fastparse/internal/MacroRepImpls.scala | 2 +- fastparse/src/fastparse/ParsingRun.scala | 26 +++++++++++++---- fastparse/src/fastparse/internal/Util.scala | 28 ++++++++++-------- .../test/src/fastparse/FailureTests.scala | 29 ++++++++++++++++++- .../src/scalaparse/unit/FailureTests.scala | 2 +- 6 files changed, 71 insertions(+), 24 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index 91500c36..560cf363 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -41,7 +41,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) res }else { val beforeSepIndex = ctx.index @@ -102,7 +102,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) res }else { val beforeSepIndex = ctx.index @@ -163,7 +163,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) res } else { val beforeSepIndex = ctx.index @@ -231,7 +231,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) reportParseMsgInRep(startIndex, min, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) res }else{ val beforeSepIndex = ctx.index diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index b72959ed..bc8048bf 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -105,7 +105,7 @@ object MacroRepImpls { val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) Util.reportParseMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut) + if (verboseFailures) Util.reportParseMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) res } else { val beforeSepIndex = ctx.index diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index edc8d4fc..05766e44 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -230,10 +230,18 @@ final class ParsingRun[+T](val input: ParserInput, } /** - * Called by any terminal parser; these are the smallest parsers that a user - * may care about, e.g. individual strings or characters, and will be stored - * in the `failureTerminals` in case a user wants to know what could - * have been placed at the failure point to let the parse progress + * Called by any terminal parser; these are parsers for which displaying + * sub-failures does not make sense these include: + * + * - Individual strings or characters + * - Parsers like negation `!p` or `.filter` where the entire parser failing + * is not caused by sub-failure + * - Parsers like `.opaque`, where sub-failures are intentionally hidden and + * not shown to the user + * + * These "terminal" failures will be stored in the `failureTerminals` in case + * a user wants to know what could have been placed at the failure point to + * let the parse progress */ def reportTerminalParseMsg(startIndex: Int, newShortParserMsg: Msgs): Unit = { @@ -254,7 +262,15 @@ final class ParsingRun[+T](val input: ParserInput, newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { - +// println() +// println(" " * logDepth + "reportParseMsg0 " + startIndex) +// println(" " * logDepth + "newShortParserMsg " + newShortParserMsg) +// println(" " * logDepth + "newFailureGroups " + newFailureGroups) +// println(" " * logDepth + "forceAggregate " + forceAggregate) +// println(" " * logDepth + "setShortMsg " + setShortMsg) +// println(" " * logDepth + "shortParserMsg " + shortParserMsg) +// println(" " * logDepth + "failureGroups " + failureGroups) +// println() if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 08daf7f4..71fc03df 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -104,25 +104,29 @@ object Util { } - def reportParseMsgPostSep[V](startIndex: Int, - min: Int, - ctx: ParsingRun[Any], - parsedMsg: Msgs, - lastAgg: Msgs) = { + def reportParseMsgPostSep(startIndex: Int, + min: Int, + ctx: ParsingRun[Any], + parsedMsg: Msgs, + lastAgg: Msgs) = { reportParseMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) } - def reportParseMsgInRep[V](startIndex: Int, - min: Int, - ctx: ParsingRun[Any], - sepMsg: Msgs, - parsedMsg: Msgs, - lastAgg: Msgs, - precut: Boolean) = { + def reportParseMsgInRep(startIndex: Int, + min: Int, + ctx: ParsingRun[Any], + sepMsg: Msgs, + parsedMsg: Msgs, + lastAgg: Msgs, + precut: Boolean) = { // When we fail on a rep body, we collect both the concatenated // sep and failure aggregate of the rep body that we tried (because // we backtrack past the sep on failure) as well as the failure // aggregate of the previous rep, which we could have continued + println("reportParseMsgInRep") + println("sepMsg " + sepMsg) + println("precut " + precut) + println("parsedMsg " + parsedMsg) val newAgg = if (sepMsg == null || precut) ctx.failureGroups else Util.joinBinOp(sepMsg, parsedMsg) diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index e0c4c913..95189c0b 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -16,9 +16,10 @@ object FailureTests extends TestSuite{ val trace = f.trace(true) val terminals1 = Option(terminals).getOrElse(expected) + val groupAggregateString = trace.groupAggregateString assert( trace.failure.label == label, - trace.groupAggregateString == expected, + groupAggregateString == expected, trace.terminalAggregateString == terminals1 ) } @@ -219,6 +220,31 @@ object FailureTests extends TestSuite{ parseA(_) } ) + test("repSeparatorIsNotIncludedInFailureMsgWhenCut") - checkOffset( + input = "ab aa", + expected = "\"b\"", + label = "\"b\"", + terminals = "\"b\"", + parser = { + def space[$: P] = P(" ") + def token[$: P] = P("a" ~/ "b") + def multiple[$: P] = P(token.rep(1, space)) + multiple(_) + } + ) + test("repSeparatorIsNotIncludedInFailureMsgWhenCutX") - checkOffset( + input = "ab aa", + expected = "\"b\"", + label = "\"b\"", + terminals = "\"b\"", + parser = { + def space[$: P] = P(" ") + def token[$: P] = P("a" ~/ "b") + def multiple[$: P] = P(token.repX(1, space)) + multiple(_) + } + ) + } test("offset"){ @@ -403,6 +429,7 @@ object FailureTests extends TestSuite{ test("repXLeft") - check{ implicit c => (("a" ~ "b") ~ "c").repX ~ "a" ~/ "d" } test("repSep") - check{ implicit c => ("a" ~ ("b" ~ "c")).rep(sep = Pass) ~ "a" ~/ "d" } test("repSepLeft") - check{ implicit c => (("a" ~ "b") ~ "c").rep(sep = Pass) ~ "a" ~/ "d" } + } test("whitespace"){ diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 273c31ad..a63b3e37 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -18,7 +18,7 @@ object FailureTests extends TestSuite{ |import a |import import """.stripMargin, - aggregate = """(Semis ~ `package` | Semis ~ TopStat | ThisPath | IdPath)""", + aggregate = """(ThisPath | IdPath)""", terminals = """("this" | "super" | "`" | var-id | chars-while(OpCharNotSlash, 1) | "/" | operator | plain-id | id)""", found = "import" ) From cbe9d56e660f4fa6e6617609c8b5aecdeb94be49 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 01:18:27 +0800 Subject: [PATCH 18/49] fix broken tests --- fastparse/src/fastparse/internal/Util.scala | 4 ---- scalaparse/test/src/scalaparse/unit/FailureTests.scala | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 71fc03df..e492ed15 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -123,10 +123,6 @@ object Util { // sep and failure aggregate of the rep body that we tried (because // we backtrack past the sep on failure) as well as the failure // aggregate of the previous rep, which we could have continued - println("reportParseMsgInRep") - println("sepMsg " + sepMsg) - println("precut " + precut) - println("parsedMsg " + parsedMsg) val newAgg = if (sepMsg == null || precut) ctx.failureGroups else Util.joinBinOp(sepMsg, parsedMsg) diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index a63b3e37..0625e4ca 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -447,7 +447,7 @@ object FailureTests extends TestSuite{ | var = 2 |} """.stripMargin, - aggregate = """(Semis ~ TmplStat | Binding | InfixPattern | VarId)""", + aggregate = """(Binding | InfixPattern | VarId)""", terminals = null, found = "= 2" ) @@ -519,7 +519,7 @@ object FailureTests extends TestSuite{ |package omg |; """.stripMargin, - aggregate = """(Semis ~ TopStat | "{")""", + aggregate = """("." | "{")""", terminals = null, found = ";" ) From f29652585fa5e82c63b11f6b60514d21a1a0b979 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 02:35:40 +0800 Subject: [PATCH 19/49] pinpointed and fixed another bug --- .../fastparse/internal/MacroInlineImpls.scala | 3 +- fastparse/src/fastparse/Parsed.scala | 4 +-- fastparse/src/fastparse/ParsingRun.scala | 9 +++--- fastparse/src/fastparse/internal/Util.scala | 29 +++++++++++++------ .../test/src/fastparse/ExampleTests.scala | 2 +- .../test/src/fastparse/FailureTests.scala | 12 +++++++- 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 2141bca7..84db7303 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -201,7 +201,7 @@ object MacroInlineImpls { } } - val guardedRhs = whitespace match { + whitespace match { case null => rhsSnippet case ws => if (ws.asTerm.tpe =:= TypeRepr.of[fastparse.NoWhitespace.noWhitespaceImplicit.type]) rhsSnippet @@ -213,7 +213,6 @@ object MacroInlineImpls { } } } - guardedRhs } } }.asInstanceOf[ParsingRun[R]] diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index dfe5ee22..9175e80c 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -180,7 +180,7 @@ object Parsed{ def groupAggregateString = groups.render @deprecated("Use .msg instead") - def trace = reportParseMsg + def trace = aggregateMsg /** * Displays the short failure message excluding the parse stack. This shows * the last parser which failed causing the parse to fail. Note that this @@ -203,7 +203,7 @@ object Parsed{ * at the failure index. This gives you a good high-level overview of what * the parser expected, at the cost */ - def reportParseMsg = Failure.formatMsg(input, List(groupAggregateString -> index), index) + def aggregateMsg = Failure.formatMsg(input, List(groupAggregateString -> index), index) /** * A version of [[msg]] that includes the parse stack diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 05766e44..68487873 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -264,12 +264,13 @@ final class ParsingRun[+T](val input: ParserInput, setShortMsg: Boolean): Unit = { // println() // println(" " * logDepth + "reportParseMsg0 " + startIndex) -// println(" " * logDepth + "newShortParserMsg " + newShortParserMsg) -// println(" " * logDepth + "newFailureGroups " + newFailureGroups) +// println(" " * logDepth + "isSuccess " + isSuccess) +// println(" " * logDepth + "newShortParserMsg " + newShortParserMsg.value()) +// println(" " * logDepth + "newFailureGroups " + newFailureGroups.value()) // println(" " * logDepth + "forceAggregate " + forceAggregate) // println(" " * logDepth + "setShortMsg " + setShortMsg) -// println(" " * logDepth + "shortParserMsg " + shortParserMsg) -// println(" " * logDepth + "failureGroups " + failureGroups) +// println(" " * logDepth + "shortParserMsg " + shortParserMsg.value()) +// println(" " * logDepth + "failureGroups " + failureGroups.value()) // println() if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index e492ed15..7e956ac5 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -10,14 +10,14 @@ object Util { case Seq(x) => x case xs => xs.mkString("(", " | ", ")") } - def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = { - () => - if (lhs.value().isEmpty) rhs.render - else if (rhs.value().isEmpty) lhs.render - else lhs.render + " ~ " + rhs.render - } - - + def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = Msgs( + Lazy( + () => + if (lhs.value().isEmpty) rhs.value() + else if (rhs.value().isEmpty) lhs.value() + else List(lhs.render + " ~ " + rhs.render) + ) + ) def consumeWhitespace[V](whitespace: fastparse.Whitespace, ctx: ParsingRun[Any]) = { val oldCapturing = ctx.noDropBuffer // completely disallow dropBuffer @@ -109,6 +109,7 @@ object Util { ctx: ParsingRun[Any], parsedMsg: Msgs, lastAgg: Msgs) = { +// println("reportParseMsgPostSep") reportParseMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) } @@ -119,6 +120,15 @@ object Util { parsedMsg: Msgs, lastAgg: Msgs, precut: Boolean) = { +// println("reportParseMsgInRep") +// println(s"startIndex $startIndex") +// println(s"min $min") +// println(s"sepMsg ${sepMsg.value()}") +// println(s"parsedMsg ${parsedMsg.value()}") +// println(s"lastAgg ${lastAgg.value()}") +// println(s"precut $precut") +// println(s"ctx.failureGroups ${ctx.failureGroups.value()}") + // When we fail on a rep body, we collect both the concatenated // sep and failure aggregate of the rep body that we tried (because // we backtrack past the sep on failure) as well as the failure @@ -126,7 +136,8 @@ object Util { val newAgg = if (sepMsg == null || precut) ctx.failureGroups else Util.joinBinOp(sepMsg, parsedMsg) - +// println(s"newAgg ${newAgg.value()}") +// println(s"newAgg ::: lastAgg ${(newAgg ::: lastAgg).value()}") ctx.reportParseMsg( startIndex, () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index 765d7253..9e29d328 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -55,7 +55,7 @@ object ExampleTests extends TestSuite{ assert( - trace.reportParseMsg == """Expected (parseEither | "c"):1:1, found "d"""", + trace.aggregateMsg == """Expected (parseEither | "c"):1:1, found "d"""", trace.longAggregateMsg == """Expected parseA:1:1 / (parseEither | "c"):1:1, found "d"""" ) } diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 95189c0b..9238a67e 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -244,7 +244,17 @@ object FailureTests extends TestSuite{ multiple(_) } ) - + test("repSeparatorsBeforeTraceIndexDontPolluteFailureGroups") - checkOffset( + input = "p ii", + expected = "\"a\"", + label = "\"a\"", + terminals = "\"a\"", + parser = { + def space[$:P] = P( " " ) + def items[$: P]: P[Unit] = P( "p".rep(sep = " ").log("p.rep") ~ space ~ "i" ~ "a" ) + items(_) + } + ) } test("offset"){ From 3c1bfc3b193057160e405ab0bbcbbeefa12edea0 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 03:25:57 +0800 Subject: [PATCH 20/49] another fix to try and limit unwanted failure group collection --- fastparse/src/fastparse/ParsingRun.scala | 13 +++++++++++-- fastparse/test/src/fastparse/FailureTests.scala | 14 +++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 68487873..78aa5d26 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -216,17 +216,26 @@ final class ParsingRun[+T](val input: ParserInput, reportParseMsg(startIndex, newShortParserMsg, newFailureGroups, false) } + /** + * We only want to set the shortMsg for most parsers if they could have + * potentially extended passed the [[traceIndex]], since that is the point at + * which all error reporting in Fastparse is focused. That means we want + * parsers that have either succeeded past the traceIndex, or failed and + * potentially backtracked. + */ + def shouldSetShortMsg = true//!isSuccess || index >= traceIndex + def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, true) + reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, shouldSetShortMsg) } def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) + reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, shouldSetShortMsg) } /** diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 9238a67e..f158934f 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -251,10 +251,22 @@ object FailureTests extends TestSuite{ terminals = "\"a\"", parser = { def space[$:P] = P( " " ) - def items[$: P]: P[Unit] = P( "p".rep(sep = " ").log("p.rep") ~ space ~ "i" ~ "a" ) + def items[$: P]: P[Unit] = P( "p".rep(sep = " ") ~ space ~ "i" ~ "a" ) items(_) } ) + test("repSeparatorsBeforeTraceIndexDontPolluteFailureGroups2") - checkOffset( + input = "p ii", + expected = "\"a\"", + label = "\"a\"", + terminals = "\"a\"", + parser = { + def space[$: P] = P(" ") + def prep[$: P] = P("p".rep(sep = space)) + def all[$: P] = P(prep ~ AnyChar ~ "i" ~ "a") + all(_) + } + ) } test("offset"){ From 41f3abf7fc48ec5b7caa089c70e98817cffa9151 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 03:26:13 +0800 Subject: [PATCH 21/49] another fix to try and limit unwanted failure group collection --- scalaparse/test/src/scalaparse/unit/FailureTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 0625e4ca..5a3df385 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -179,7 +179,7 @@ object FailureTests extends TestSuite{ |import org.parboiled2 _ | """.stripMargin, - aggregate = """(Semis ~ `package` | "." | "," | end-of-input)""", + aggregate = """(`package` | "." | "," | end-of-input)""", terminals = null, found = "_" ) From 9c1a8f78d6eac52dce48e34c23ad4047473734ac Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 03:26:58 +0800 Subject: [PATCH 22/49] another fix to try and limit unwanted failure group collection --- fastparse/src/fastparse/ParsingRun.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 78aa5d26..6ec460ed 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -223,7 +223,7 @@ final class ParsingRun[+T](val input: ParserInput, * parsers that have either succeeded past the traceIndex, or failed and * potentially backtracked. */ - def shouldSetShortMsg = true//!isSuccess || index >= traceIndex + def shouldSetShortMsg = !isSuccess || index >= traceIndex def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, From 55005dfdb89e5a3abc7506eda8c1d27b81719893 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 04:03:06 +0800 Subject: [PATCH 23/49] next error reporting bug minimized --- fastparse/src/fastparse/ParsingRun.scala | 20 +++++++++---------- fastparse/src/fastparse/internal/Util.scala | 2 +- .../test/src/fastparse/FailureTests.scala | 19 ++++++++++++++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 6ec460ed..6b91d5a8 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -271,16 +271,16 @@ final class ParsingRun[+T](val input: ParserInput, newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { -// println() -// println(" " * logDepth + "reportParseMsg0 " + startIndex) -// println(" " * logDepth + "isSuccess " + isSuccess) -// println(" " * logDepth + "newShortParserMsg " + newShortParserMsg.value()) -// println(" " * logDepth + "newFailureGroups " + newFailureGroups.value()) -// println(" " * logDepth + "forceAggregate " + forceAggregate) -// println(" " * logDepth + "setShortMsg " + setShortMsg) -// println(" " * logDepth + "shortParserMsg " + shortParserMsg.value()) -// println(" " * logDepth + "failureGroups " + failureGroups.value()) -// println() + println() + println(" " * logDepth + "reportParseMsg0 " + startIndex) + println(" " * logDepth + "isSuccess " + isSuccess) + println(" " * logDepth + "newShortParserMsg " + newShortParserMsg.value()) + println(" " * logDepth + "newFailureGroups " + newFailureGroups.value()) + println(" " * logDepth + "forceAggregate " + forceAggregate) + println(" " * logDepth + "setShortMsg " + setShortMsg) + println(" " * logDepth + "shortParserMsg " + shortParserMsg.value()) + println(" " * logDepth + "failureGroups " + failureGroups.value()) + println() if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 7e956ac5..82df3e4c 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -11,7 +11,7 @@ object Util { case xs => xs.mkString("(", " | ", ")") } def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = Msgs( - Lazy( + new Lazy( () => if (lhs.value().isEmpty) rhs.value() else if (rhs.value().isEmpty) lhs.value() diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index f158934f..a5ad2483 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -12,14 +12,16 @@ object FailureTests extends TestSuite{ terminals: String = null, parser: P[_] => P[_]) = { val f @ Parsed.Failure(failureString, index, extra) = parse(input, parser(_)) - + println("x" * 150) val trace = f.trace(true) + println("index " + index) + println("trace.index " + trace.index) val terminals1 = Option(terminals).getOrElse(expected) val groupAggregateString = trace.groupAggregateString assert( - trace.failure.label == label, groupAggregateString == expected, + trace.failure.label == label, trace.terminalAggregateString == terminals1 ) } @@ -267,6 +269,19 @@ object FailureTests extends TestSuite{ all(_) } ) + test("bug") - checkOffset( + input = "pt x_", + expected = """("y" | end-of-input)""", + label = "\"a\"", + terminals = "\"a\"", + parser = { + def c[$: P] = P( "x".repX(1, "y") ).log + def d[$: P] = P( "p" ).log + def b[$: P] = P( (d ~ "t").repX(1, " ") ).log + def a[$: P] = P( b ~ " " ~ c ~ End ).log + a(_) + } + ) } test("offset"){ From a4516bf8482fe4a8252592738c23d7557bb3c459 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 05:42:20 +0800 Subject: [PATCH 24/49] Overhaul handling of parse reporting. --- .../fastparse/internal/MacroInlineImpls.scala | 29 ++++++----- .../fastparse/internal/MacroRepImpls.scala | 12 ++++- fastparse/src/fastparse/Parsed.scala | 2 +- fastparse/src/fastparse/ParsingRun.scala | 30 ++++++------ .../src/fastparse/SharedPackageDefs.scala | 15 +++--- fastparse/src/fastparse/Whitespace.scala | 6 +-- .../test/src/fastparse/FailureTests.scala | 12 +++-- .../src/scalaparse/syntax/Literals.scala | 6 +-- .../src/scalaparse/unit/FailureTests.scala | 48 +++++++++---------- 9 files changed, 91 insertions(+), 69 deletions(-) diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 84db7303..b1a1447d 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -187,16 +187,20 @@ object MacroInlineImpls { ) } - if (ctx1.verboseFailures) ctx1.reportParseMsg( - preLhsIndex, - Util.joinBinOp(lhsMsg, rhsMsg), - rhsAggregate ::: lhsAggregate, - // We override the failureGroups to avoid building an `a ~ b` - // aggregate msg in the specific case where the LHS parser fails to - // make any progress past `startIndex`. This finds cases like `a.? ~ b` - // or `a.rep ~ b` and lets use flatten them out into `a | b` - forceAggregate = preRhsIndex == ctx1.traceIndex - ) + if (ctx1.verboseFailures) { +// println("rhsAggregate " + rhsAggregate) +// println("lhsAggregate " + lhsAggregate) + ctx1.reportParseMsg( + preLhsIndex, + Util.joinBinOp(lhsMsg, rhsMsg), + rhsAggregate ::: lhsAggregate, + // We override the failureGroups to avoid building an `a ~ b` + // aggregate msg in the specific case where the LHS parser fails to + // make any progress past `startIndex`. This finds cases like `a.? ~ b` + // or `a.rep ~ b` and lets use flatten them out into `a | b` + forceAggregate = preRhsIndex == ctx1.traceIndex + ) + } res } } @@ -316,8 +320,9 @@ object MacroInlineImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) - ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) + if (verboseFailures) { + ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) + } ctx5.asInstanceOf[ParsingRun[V]] } } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index bc8048bf..cfd285fe 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -105,7 +105,17 @@ object MacroRepImpls { val res = if (postCut) ctx.asInstanceOf[ParsingRun[V]] else end(startIndex, startIndex, count, outerCut | postCut) - if (verboseFailures) Util.reportParseMsgInRep(startIndex, actualMin, ctx, sepMsg, parsedMsg, lastAgg, precut || postCut) + if (verboseFailures) { + Util.reportParseMsgInRep( + startIndex, + actualMin, + ctx, + sepMsg, + parsedMsg, + lastAgg, + precut || postCut + ) + } res } else { val beforeSepIndex = ctx.index diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index 9175e80c..3b3983ee 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -149,7 +149,7 @@ object Parsed{ assert(!p.isSuccess) TracedFailure( p.failureTerminals, - p.lastFailureMsg ::: p.failureGroups, + p.failureGroups, Parsed.fromParsingRun(p).asInstanceOf[Failure] ) } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 6b91d5a8..acb7fff6 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -223,7 +223,7 @@ final class ParsingRun[+T](val input: ParserInput, * parsers that have either succeeded past the traceIndex, or failed and * potentially backtracked. */ - def shouldSetShortMsg = !isSuccess || index >= traceIndex + def shouldSetShortMsg = failureGroups.value().nonEmpty def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, @@ -235,7 +235,7 @@ final class ParsingRun[+T](val input: ParserInput, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, shouldSetShortMsg) + reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, newFailureGroups.value().nonEmpty) } /** @@ -259,28 +259,30 @@ final class ParsingRun[+T](val input: ParserInput, reportParseMsg0( startIndex, - newShortParserMsg, - Msgs.empty, + if (startIndex >= traceIndex) newShortParserMsg else Msgs.empty, + if (startIndex >= traceIndex) newShortParserMsg else Msgs.empty, false, startIndex >= traceIndex ) } + def logStuff(s: String) = { + println() + println(" " * logDepth + s + " reportParseMsg0 " + index) + println(" " * logDepth + s + " isSuccess " + isSuccess) + println(" " * logDepth + s + " shortParserMsg " + shortParserMsg.value()) + println(" " * logDepth + s + " failureGroups " + failureGroups.value()) + println() + } def reportParseMsg0(startIndex: Int, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { - println() - println(" " * logDepth + "reportParseMsg0 " + startIndex) - println(" " * logDepth + "isSuccess " + isSuccess) - println(" " * logDepth + "newShortParserMsg " + newShortParserMsg.value()) - println(" " * logDepth + "newFailureGroups " + newFailureGroups.value()) - println(" " * logDepth + "forceAggregate " + forceAggregate) - println(" " * logDepth + "setShortMsg " + setShortMsg) - println(" " * logDepth + "shortParserMsg " + shortParserMsg.value()) - println(" " * logDepth + "failureGroups " + failureGroups.value()) - println() + // println(" " * logDepth + s + "newShortParserMsg " + newShortParserMsg.value()) + // println(" " * logDepth + s + "newFailureGroups " + newFailureGroups.value()) + // println(" " * logDepth + s + "forceAggregate " + forceAggregate) + // println(" " * logDepth + s + "setShortMsg " + setShortMsg) if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 4955152d..91e96376 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -106,10 +106,13 @@ trait SharedPackageDefs { else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { ctx.failureGroups = Msgs.empty - ctx.reportTerminalParseMsg(startPos, () => - msg match{ + ctx.reportTerminalParseMsg(startPos, + if (msg.value().isEmpty) Msgs.empty + else () => msg match{ case Seq(x) => s"&(${msg.render})" - case xs => s"&${msg.render}" + case xs => + new Exception().printStackTrace() + s"&${msg.render}" } ) } @@ -167,7 +170,7 @@ trait SharedPackageDefs { */ def Pass(implicit ctx: P[_]): P[Unit] = { val res = ctx.freshSuccessUnit() - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Pass") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, Msgs.empty) res } @@ -211,7 +214,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Unit]] else ctx.freshSuccessUnit(ctx.index + 1) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-char") res } @@ -227,7 +230,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Char]] else ctx.freshSuccess(ctx.input(ctx.index), ctx.index + 1) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-character") + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-char") res } diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index c0643bd6..2f41b7ce 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) res } } else { @@ -153,7 +153,7 @@ object JsonnetWhitespace{ ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) res } } else { @@ -205,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) res } } else { diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index a5ad2483..f166a9fc 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -19,10 +19,12 @@ object FailureTests extends TestSuite{ val terminals1 = Option(terminals).getOrElse(expected) val groupAggregateString = trace.groupAggregateString + val traceLabel = trace.failure.label + val traceTerminalAggregateString = trace.terminalAggregateString assert( groupAggregateString == expected, - trace.failure.label == label, - trace.terminalAggregateString == terminals1 + traceLabel == label, + traceTerminalAggregateString == terminals1 ) } @@ -269,11 +271,11 @@ object FailureTests extends TestSuite{ all(_) } ) - test("bug") - checkOffset( + test("repSeparatorsBeforeTraceIndexDontPolluteFailureGroups3") - checkOffset( input = "pt x_", expected = """("y" | end-of-input)""", - label = "\"a\"", - terminals = "\"a\"", + label = "end-of-input", + terminals = """("y" | end-of-input)""", parser = { def c[$: P] = P( "x".repX(1, "y") ).log def d[$: P] = P( "p" ).log diff --git a/scalaparse/src/scalaparse/syntax/Literals.scala b/scalaparse/src/scalaparse/syntax/Literals.scala index 14e8985f..f6366747 100644 --- a/scalaparse/src/scalaparse/syntax/Literals.scala +++ b/scalaparse/src/scalaparse/syntax/Literals.scala @@ -58,7 +58,7 @@ trait Literals { l => def Null[$: P] = Key.W("null") def OctalEscape[$: P] = P( Digit ~ Digit.? ~ Digit.? ) - def Escape[$: P] = P( "\\" ~/ (CharIn("""btnfr'\\"]""") | OctalEscape | UnicodeEscape ) ) + def Escape[$: P] = P( "\\" ~/ (CharIn("""btnfr'\\"]""") | OctalEscape | UnicodeEscape ) ).log // Note that symbols can take on the same values as keywords! def Symbol[$: P] = P( Identifiers.PlainId | Identifiers.Keywords ) @@ -93,6 +93,7 @@ trait Literals { l => def NonStringEnd = P( !CharIn("\n\"") ~ AnyChar ) P( (StringChars | Interp | LiteralSlash | Escape | NonStringEnd ).rep ) } + def String[$: P] = { P { Id.filter(_ => interp.isDefined) ~ ( @@ -103,11 +104,10 @@ trait Literals { l => "\"" ~/ NoInterp.SingleChars(false) ~ "\"" } } - } + def NoInterp[$: P] = new InterpCtx(None) def Pat[$: P] = new InterpCtx(Some(() => l.Pattern)) def Expr[$: P] = new InterpCtx(Some(() => Block)) - } } diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 5a3df385..66ee2784 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -70,7 +70,7 @@ object FailureTests extends TestSuite{ | } |} """.stripMargin, - aggregate = """(FunArgs | `:` | Body | Semis | "}")""", + aggregate = """(FunArgs | `:` | Body | "}")""", terminals = null, found = "](input: S" ) @@ -93,7 +93,7 @@ object FailureTests extends TestSuite{ | } |} """.stripMargin, - aggregate = """("=>" | `:` | "." | TypeArgs | ArgList | `_` | Id | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """("=>" | `:` | "." | TypeArgs | ArgList | `_` | Id | `=` | MatchAscriptionSuffix | &"}")""", terminals = null, found ="1\n" ) @@ -117,7 +117,7 @@ object FailureTests extends TestSuite{ | filename.asInstanceOf 10 |} """.stripMargin, - aggregate = """("." | TypeArgs | ArgList | `_` | Id | "=>" | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """("." | TypeArgs | ArgList | `_` | Id | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = "10" ) @@ -179,7 +179,7 @@ object FailureTests extends TestSuite{ |import org.parboiled2 _ | """.stripMargin, - aggregate = """(`package` | "." | "," | end-of-input)""", + aggregate = """("." | "," | end-of-input)""", terminals = null, found = "_" ) @@ -220,7 +220,7 @@ object FailureTests extends TestSuite{ |} | """.stripMargin, - aggregate = """(WL ~ "." | WL ~ TypeArgs | NotNewline ~ ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """("." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = ")" ) @@ -251,7 +251,7 @@ object FailureTests extends TestSuite{ | d = 1 | """.stripMargin, - aggregate = """("." | TypeArgs | "=>" | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """("." | TypeArgs | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = "" ) @@ -361,7 +361,7 @@ object FailureTests extends TestSuite{ | a =:= .c |} """.stripMargin, - aggregate = """(TypeArgs | PrefixExpr | Newline | "=>" | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """(TypeArgs | PrefixExpr | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = ".c" ) @@ -416,7 +416,7 @@ object FailureTests extends TestSuite{ | val trueA = 1 |} """.stripMargin, - aggregate = """(DefTmpl | Semis ~ TopStat | end-of-input)""", + aggregate = """(DefTmpl | TopStat | end-of-input)""", terminals = null, found = "val trueA" ) @@ -426,7 +426,7 @@ object FailureTests extends TestSuite{ | val null null cow = 1 |} """.stripMargin, - aggregate = """(Id | "," | `:` | `=` | Semis | "}")""", + aggregate = """(Id | "," | `:` | `=` | "}")""", terminals = null, found = "null cow" ) @@ -436,7 +436,7 @@ object FailureTests extends TestSuite{ | val omg_+_+ = 1 |} """.stripMargin, - aggregate = """(`@` | TQ | "\"" | "." | TypeArgs | TupleEx | Id | "," | `:` | `=` | Semis | "}")""", + aggregate = """(`@` | TQ | "\"" | "." | TypeArgs | TupleEx | Id | "," | `:` | `=` | "}")""", terminals = null, found = "_+ = 1" ) @@ -480,7 +480,7 @@ object FailureTests extends TestSuite{ | a!.b |} """.stripMargin, - aggregate = """(TypeArgs | PrefixExpr | Newline | "=>" | `=` | MatchAscriptionSuffix | Semis | "}")""", + aggregate = """(TypeArgs | PrefixExpr | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = ".b" ) @@ -529,7 +529,7 @@ object FailureTests extends TestSuite{ | { a: L = } |} """.stripMargin, - aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | "=>" | BlockLambda | BlockStat | Semis | "}")""", + aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | "=>" | BlockLambda | BlockStat | &"}")""", terminals = null, found = "= }" ) @@ -549,7 +549,7 @@ object FailureTests extends TestSuite{ |} | """.stripMargin, - aggregate = """(PostDotCheck | id)""", + aggregate = """id""", terminals = null, found = "this" ) @@ -645,7 +645,7 @@ object FailureTests extends TestSuite{ |} | """.stripMargin, - aggregate = """(TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | "," ~ Type | "," ~ WS ~ Newline | "]")""", + aggregate = """(TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | "," ~ Type | "," | "]")""", terminals = null, found = ", ]" ) @@ -669,7 +669,7 @@ object FailureTests extends TestSuite{ | } |} """.stripMargin, - aggregate = """(BlockLambda | BlockStat | Semis | "}")""", + aggregate = """(BlockLambda | BlockStat | &"}")""", terminals = null, found = "case for" ) @@ -691,7 +691,7 @@ object FailureTests extends TestSuite{ |} | """.stripMargin, - aggregate = """(StringChars | Interp | LiteralSlash | Escape | NonStringEnd | "\"")""", + aggregate = """(StringChars | Interp | LiteralSlash | Escape | "\"")""", terminals = null, found = "\n" ) @@ -759,7 +759,7 @@ object FailureTests extends TestSuite{ | val (x,) = 1 |} """.stripMargin, - aggregate = """(`:` | `@` | TQ | "\"" | "." | TypeArgs | TupleEx | Id | "|" | "," ~ Pattern | "," ~ WS ~ Newline | ")")""", + aggregate = """(`:` | `@` | TQ | "\"" | "." | TypeArgs | TupleEx | Id | "|" | "," ~ Pattern | "," | ")")""", terminals = null, found = ",)" ) @@ -799,7 +799,7 @@ object FailureTests extends TestSuite{ s""" |object X{def f(x: Int, ) = 1} """.stripMargin, - aggregate = """("." | TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | `=` | "," ~ FunArg | "," ~ WS ~ Newline | ")")""", + aggregate = """("." | TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | `=` | "," ~ FunArg | "," | ")")""", terminals = null, found = ", )" ) @@ -807,7 +807,7 @@ object FailureTests extends TestSuite{ s""" |object X{(2,)} """.stripMargin, - aggregate = """(FloatSuffix | "L" | "l" | WL ~ "." | WL ~ TypeArgs | Pass ~ ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "," ~ Expr | "," ~ WS ~ Newline | ")")""", + aggregate = """(FloatSuffix | "L" | "l" | WL ~ "." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "," ~ Expr | "," | ")")""", terminals = null, found = ",)" ) @@ -815,7 +815,7 @@ object FailureTests extends TestSuite{ s""" |object X{f[A,]} """.stripMargin, - aggregate = """("." | TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | "," ~ Type | "," ~ WS ~ Newline | "]")""", + aggregate = """("." | TypeArgs | `#` | NLAnnot | `with` | Refinement | `*` | Id | "=>" | ExistentialClause | `>:` | `<:` | "," ~ Type | "," | "]")""", terminals = null, found = ",]" ) @@ -831,7 +831,7 @@ object FailureTests extends TestSuite{ s""" |object X{def f[T, B,] = 1} """.stripMargin, - aggregate = """(TypeArgList | `>:` | `<:` | `<%` | `:` | "," ~ Annot.rep ~ TypeArg | "," ~ WS ~ Newline | "]")""", + aggregate = """(TypeArgList | `>:` | `<:` | `<%` | `:` | "," ~ Annot.rep ~ TypeArg | "," | "]")""", terminals = null, found = ",]" ) @@ -943,13 +943,13 @@ object FailureTests extends TestSuite{ | for(i <- Nil if x: Int => bar) 1 |} """.stripMargin, - aggregate = """(TQ | "\"" | "." | WL ~ "." | WL ~ TypeArgs | Pass ~ ArgList | `_` | InfixSuffix | PostFix | Enumerator | ")")""", + aggregate = """(TQ | "\"" | "." | WL ~ "." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | Enumerator | ")")""", terminals = null, found = ": Int" ) test - checkNeg( s"""object Foo{; x: Int => x}""", - aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | Semis | "}")""", + aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | "}")""", terminals = null, found = "=> x" ) @@ -978,7 +978,7 @@ object FailureTests extends TestSuite{ | val x = 1 | ; | """.stripMargin, - aggregate = """(BlockLambda | BlockStat | Semis | "}")""", + aggregate = """(BlockLambda | BlockStat | &"}")""", terminals = null, found = "" ) From 10fd7512bd6b8a2da76b2d5e6f8cbc068ea1f1e3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 05:53:07 +0800 Subject: [PATCH 25/49] revert Msgs from Lazy[List] back to List[Lazy] --- .../src-2/fastparse/internal/MacroImpls.scala | 2 +- .../fastparse/internal/MacroInlineImpls.scala | 2 +- fastparse/src/fastparse/ParsingRun.scala | 12 ++++---- .../src/fastparse/SharedPackageDefs.scala | 2 +- fastparse/src/fastparse/internal/Util.scala | 29 +++++++++---------- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index ca1e4c24..2530d86d 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -362,7 +362,7 @@ object MacroImpls { val res = if ($output != -1) $ctx1.freshSuccessUnit(index = $output) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($index, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($index, $bracketed) res } """ diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index b1a1447d..6699c7d0 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -544,7 +544,7 @@ object MacroInlineImpls { else ctx1.freshFailure() if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg( index, - () => ${ Expr(literals.map(Util.literalize(_)).toList) } + Msgs.fromStrings(${ Expr(literals.map(Util.literalize(_)).toList) }) ) res } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index acb7fff6..14053f20 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -223,7 +223,7 @@ final class ParsingRun[+T](val input: ParserInput, * parsers that have either succeeded past the traceIndex, or failed and * potentially backtracked. */ - def shouldSetShortMsg = failureGroups.value().nonEmpty + def shouldSetShortMsg = failureGroups.value.nonEmpty def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, @@ -235,7 +235,7 @@ final class ParsingRun[+T](val input: ParserInput, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, newFailureGroups.value().nonEmpty) + reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, newFailureGroups.value.nonEmpty) } /** @@ -268,10 +268,10 @@ final class ParsingRun[+T](val input: ParserInput, def logStuff(s: String) = { println() - println(" " * logDepth + s + " reportParseMsg0 " + index) - println(" " * logDepth + s + " isSuccess " + isSuccess) - println(" " * logDepth + s + " shortParserMsg " + shortParserMsg.value()) - println(" " * logDepth + s + " failureGroups " + failureGroups.value()) +// println(" " * logDepth + s + " reportParseMsg0 " + index) +// println(" " * logDepth + s + " isSuccess " + isSuccess) +// println(" " * logDepth + s + " shortParserMsg " + shortParserMsg.value()) +// println(" " * logDepth + s + " failureGroups " + failureGroups.value()) println() } def reportParseMsg0(startIndex: Int, diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 91e96376..54ae75dc 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -107,7 +107,7 @@ trait SharedPackageDefs { if (ctx.verboseFailures) { ctx.failureGroups = Msgs.empty ctx.reportTerminalParseMsg(startPos, - if (msg.value().isEmpty) Msgs.empty + if (msg.value.isEmpty) Msgs.empty else () => msg match{ case Seq(x) => s"&(${msg.render})" case xs => diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 82df3e4c..6d7313f1 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -6,18 +6,15 @@ import scala.annotation.{switch, tailrec} import scala.collection.mutable.ArrayBuffer object Util { - def parenthize(fs: Lazy[List[String]]) = fs().reverseIterator.toSeq.distinct match{ + def parenthize(fs: List[Lazy[String]]) = fs.reverseIterator.map(_()).toSeq.distinct match{ case Seq(x) => x case xs => xs.mkString("(", " | ", ")") } - def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = Msgs( - new Lazy( - () => - if (lhs.value().isEmpty) rhs.value() - else if (rhs.value().isEmpty) lhs.value() - else List(lhs.render + " ~ " + rhs.render) - ) - ) + def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = { + if (lhs.value.isEmpty) rhs + else if (rhs.value.isEmpty) lhs + else Msgs.fromFunction(() => lhs.render + " ~ " + rhs.render) + } def consumeWhitespace[V](whitespace: fastparse.Whitespace, ctx: ParsingRun[Any]) = { val oldCapturing = ctx.noDropBuffer // completely disallow dropBuffer @@ -194,18 +191,18 @@ final class CompactTrieNode(source: TrieNode){ val word = source.word } object Msgs{ - val empty = Msgs(new Lazy(() => Nil)) + val empty = Msgs(Nil) implicit def fromFunction(msgToSet: () => String): Msgs = { - Msgs(new Lazy(() => msgToSet() :: Nil)) + Msgs(new Lazy(() => msgToSet()):: Nil) } - implicit def fromListFunction(msgsToSet: () => List[String]): Msgs = { - Msgs(new Lazy(msgsToSet)) + implicit def fromStrings(msgsToSet: List[String]): Msgs = { + Msgs(msgsToSet.map(s => new Lazy(() => s))) } } -case class Msgs(value: Lazy[List[String]]){ - def :::(other: Msgs) = Msgs(new Lazy(() => other.value() ::: value())) - def ::(other: Lazy[String]) = Msgs(new Lazy(() => other() :: value())) +case class Msgs(value: List[Lazy[String]]){ + def :::(other: Msgs) = Msgs(other.value ::: value) + def ::(other: Lazy[String]) = Msgs(other :: value) override def toString = render def render = Util.parenthize(value) } \ No newline at end of file From eddee31ff90df23fe5a80bd0689ad4427fa13bea Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 05:57:53 +0800 Subject: [PATCH 26/49] . --- fastparse/src/fastparse/SharedPackageDefs.scala | 4 +--- fastparse/src/fastparse/internal/Util.scala | 12 +----------- fastparse/test/src/fastparse/ExampleTests.scala | 2 +- fastparse/test/src/fastparse/FailureTests.scala | 3 --- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 54ae75dc..43775103 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -110,9 +110,7 @@ trait SharedPackageDefs { if (msg.value.isEmpty) Msgs.empty else () => msg match{ case Seq(x) => s"&(${msg.render})" - case xs => - new Exception().printStackTrace() - s"&${msg.render}" + case xs => s"&${msg.render}" } ) } diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 6d7313f1..2d0c1250 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -106,7 +106,6 @@ object Util { ctx: ParsingRun[Any], parsedMsg: Msgs, lastAgg: Msgs) = { -// println("reportParseMsgPostSep") reportParseMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) } @@ -117,14 +116,6 @@ object Util { parsedMsg: Msgs, lastAgg: Msgs, precut: Boolean) = { -// println("reportParseMsgInRep") -// println(s"startIndex $startIndex") -// println(s"min $min") -// println(s"sepMsg ${sepMsg.value()}") -// println(s"parsedMsg ${parsedMsg.value()}") -// println(s"lastAgg ${lastAgg.value()}") -// println(s"precut $precut") -// println(s"ctx.failureGroups ${ctx.failureGroups.value()}") // When we fail on a rep body, we collect both the concatenated // sep and failure aggregate of the rep body that we tried (because @@ -133,8 +124,7 @@ object Util { val newAgg = if (sepMsg == null || precut) ctx.failureGroups else Util.joinBinOp(sepMsg, parsedMsg) -// println(s"newAgg ${newAgg.value()}") -// println(s"newAgg ::: lastAgg ${(newAgg ::: lastAgg).value()}") + ctx.reportParseMsg( startIndex, () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index 9e29d328..05ca749a 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -50,7 +50,7 @@ object ExampleTests extends TestSuite{ trace.longMsg == """Expected parseA:1:1 / "c":1:1, found "d"""" ) - // reportParseMsg and longAggregateMsg record all parsers + // aggregateMsg and longAggregateMsg record all parsers // failing at the position, "a" | "b" | "c", diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index f166a9fc..cda4ba68 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -12,10 +12,7 @@ object FailureTests extends TestSuite{ terminals: String = null, parser: P[_] => P[_]) = { val f @ Parsed.Failure(failureString, index, extra) = parse(input, parser(_)) - println("x" * 150) val trace = f.trace(true) - println("index " + index) - println("trace.index " + trace.index) val terminals1 = Option(terminals).getOrElse(expected) val groupAggregateString = trace.groupAggregateString From aae6a7125e02f98f59c63848786bdc84add1d133 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 05:58:07 +0800 Subject: [PATCH 27/49] . --- fastparse/src/fastparse/ParsingRun.scala | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 14053f20..73e13045 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -266,23 +266,11 @@ final class ParsingRun[+T](val input: ParserInput, ) } - def logStuff(s: String) = { - println() -// println(" " * logDepth + s + " reportParseMsg0 " + index) -// println(" " * logDepth + s + " isSuccess " + isSuccess) -// println(" " * logDepth + s + " shortParserMsg " + shortParserMsg.value()) -// println(" " * logDepth + s + " failureGroups " + failureGroups.value()) - println() - } def reportParseMsg0(startIndex: Int, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { - // println(" " * logDepth + s + "newShortParserMsg " + newShortParserMsg.value()) - // println(" " * logDepth + s + "newFailureGroups " + newFailureGroups.value()) - // println(" " * logDepth + s + "forceAggregate " + forceAggregate) - // println(" " * logDepth + s + "setShortMsg " + setShortMsg) if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty From 9a5622768bc3c20be8ea186447db25d93e702aab Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 06:11:42 +0800 Subject: [PATCH 28/49] simplify --- fastparse/src/fastparse/ParsingRun.scala | 6 ++---- fastparse/src/fastparse/SharedPackageDefs.scala | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 73e13045..dde9d164 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -223,19 +223,17 @@ final class ParsingRun[+T](val input: ParserInput, * parsers that have either succeeded past the traceIndex, or failed and * potentially backtracked. */ - def shouldSetShortMsg = failureGroups.value.nonEmpty - def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, shouldSetShortMsg) + reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, failureGroups.value.nonEmpty) } def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, newFailureGroups.value.nonEmpty) + reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) } /** diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 43775103..50023382 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -285,7 +285,7 @@ object SharedPackageDefs{ if (ctx.verboseFailures) { ctx.failureTerminals = startTerminals ctx.failureGroups = Msgs.empty - ctx.reportParseMsg(startPos, () => "!" + msg.render, Msgs.empty) + ctx.reportTerminalParseMsg(startPos, Msgs.empty) } res.cut = startCut res From 36f37b8aaaee559140fed3022bd2c7f512a45fd2 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 06:12:21 +0800 Subject: [PATCH 29/49] simplify --- .../src-2/fastparse/internal/MacroImpls.scala | 18 +++++++------- .../src-2/fastparse/internal/RepImpls.scala | 4 ++-- .../fastparse/internal/MacroInlineImpls.scala | 18 +++++++------- .../fastparse/internal/MacroRepImpls.scala | 2 +- fastparse/src/fastparse/ParsingRun.scala | 6 ++--- .../src/fastparse/SharedPackageDefs.scala | 24 +++++++++---------- fastparse/src/fastparse/Whitespace.scala | 6 ++--- 7 files changed, 39 insertions(+), 39 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 2530d86d..9a3532f1 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -29,7 +29,7 @@ object MacroImpls { else if (f.splice(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(startIndex, () => "filter") res } } @@ -88,7 +88,7 @@ object MacroImpls { }else{ ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => literalized.splice) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => literalized.splice) res } @@ -118,7 +118,7 @@ object MacroImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.reportTerminalParseMsg(index, () => literalized.splice) + ctx1.reportTerminalMsg(index, () => literalized.splice) } res @@ -133,7 +133,7 @@ object MacroImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => Util.literalize(s1)) res } } @@ -362,7 +362,7 @@ object MacroImpls { val res = if ($output != -1) $ctx1.freshSuccessUnit(index = $output) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($index, $bracketed) + if ($ctx1.verboseFailures) $ctx1.reportTerminalMsg($index, $bracketed) res } """ @@ -428,7 +428,7 @@ object MacroImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => bracketed.splice) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => bracketed.splice) res } } @@ -569,7 +569,7 @@ object MacroImpls { else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.reportTerminalMsg(startIndex, () => s"char-pred(${p0})") res } } @@ -612,7 +612,7 @@ object MacroImpls { if ($index >= $goal) $ctx1.freshSuccessUnit(index = $index) else $ctx1.freshFailure() - if ($ctx1.verboseFailures) $ctx1.reportTerminalParseMsg($start, () => $bracketed) + if ($ctx1.verboseFailures) $ctx1.reportTerminalMsg($start, () => $bracketed) res } """ @@ -640,7 +640,7 @@ object MacroImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(start, () => s"chars-while($p0, ${min.splice})") + if (ctx0.verboseFailures) ctx0.reportTerminalMsg(start, () => s"chars-while($p0, ${min.splice})") res } } diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index 560cf363..f03f27d2 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -49,7 +49,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val nextCount = count + 1 if (nextCount == actualMax) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".repX" + (if(min == 0) "" else s"($min)")) + if (verboseFailures) ctx.reportTerminalMsg(startIndex, () => parsedMsg.render + ".repX" + (if(min == 0) "" else s"($min)")) res } else { @@ -171,7 +171,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val nextCount = count + 1 if (nextCount == actualMax) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".rep" + (if(min == 0) "" else s"($min)")) + if (verboseFailures) ctx.reportTerminalMsg(startIndex, () => parsedMsg.render + ".rep" + (if(min == 0) "" else s"($min)")) res } else if (!consumeWhitespace(whitespace, ctx, false)) ctx.asInstanceOf[ParsingRun[Nothing]] diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 6699c7d0..87fac304 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -27,7 +27,7 @@ object MacroInlineImpls { } else { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => $literalized) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => $literalized) res } @@ -51,7 +51,7 @@ object MacroInlineImpls { ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } if (ctx1.verboseFailures) { - ctx1.reportTerminalParseMsg(index, () => $literalized) + ctx1.reportTerminalMsg(index, () => $literalized) } res @@ -67,7 +67,7 @@ object MacroInlineImpls { val res = if (Util.startsWith(ctx1.input, s1, index)) ctx1.freshSuccessUnit(index + s1.length) else ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => Util.literalize(s1)) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => Util.literalize(s1)) res } } @@ -83,7 +83,7 @@ object MacroInlineImpls { else if (f(ctx1.successValue.asInstanceOf[T])) ctx1.asInstanceOf[ParsingRun[T]] else ctx1.freshFailure().asInstanceOf[ParsingRun[T]] - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(startIndex, () => "filter") + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(startIndex, () => "filter") res } @@ -401,7 +401,7 @@ object MacroInlineImpls { case true => ctx1.freshSuccessUnit(index + 1) case false => ctx1.freshFailure().asInstanceOf[ParsingRun[Unit]] } - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(index, () => $bracketed) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(index, () => $bracketed) res } } @@ -424,7 +424,7 @@ object MacroInlineImpls { } else { ctx0.freshSuccessUnit(ctx0.index + 1) } - if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(startIndex, () => s"char-pred(${p0})") + if (ctx0.verboseFailures) ctx0.reportTerminalMsg(startIndex, () => s"char-pred(${p0})") res } @@ -452,7 +452,7 @@ object MacroInlineImpls { if (index >= goal) ctx1.freshSuccessUnit(index = index) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg(start, () => $bracketed) + if (ctx1.verboseFailures) ctx1.reportTerminalMsg(start, () => $bracketed) res } } @@ -467,7 +467,7 @@ object MacroInlineImpls { val res = if (index >= goal) ctx0.freshSuccessUnit(index = index) else ctx0.freshFailure() - if (ctx0.verboseFailures) ctx0.reportTerminalParseMsg(start, () => s"chars-while($p0, $min)") + if (ctx0.verboseFailures) ctx0.reportTerminalMsg(start, () => s"chars-while($p0, $min)") res } @@ -542,7 +542,7 @@ object MacroInlineImpls { val res = if (output != -1) ctx1.freshSuccessUnit(output) else ctx1.freshFailure() - if (ctx1.verboseFailures) ctx1.reportTerminalParseMsg( + if (ctx1.verboseFailures) ctx1.reportTerminalMsg( index, Msgs.fromStrings(${ Expr(literals.map(Util.literalize(_)).toList) }) ) diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index cfd285fe..359a5bce 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -129,7 +129,7 @@ object MacroRepImpls { '{ if ($checkMax2) { val res = end(beforeSepIndex, beforeSepIndex, nextCount, outerCut | postCut) - if (verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => parsedMsg.render + ".rep" + (if (actualMin == 0) "" else s"(${actualMin})")) + if (verboseFailures) ctx.reportTerminalMsg(startIndex, () => parsedMsg.render + ".rep" + (if (actualMin == 0) "" else s"(${actualMin})")) res } else { diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index dde9d164..f70b8302 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -136,7 +136,7 @@ final class ParsingRun[+T](val input: ParserInput, // the parser trying to do when it failed" // // The implementation of `failureTerminals` is straightforward: we - // simply call `reportTerminalParseMsg` in every terminal parser, which collects + // simply call `reportTerminalMsg` in every terminal parser, which collects // all the messages in a big list and returns it. The implementation of // `failureGroups` is more interesting, since we need to figure out // what are the "high level" parsers that we need to list. We use the @@ -250,8 +250,8 @@ final class ParsingRun[+T](val input: ParserInput, * a user wants to know what could have been placed at the failure point to * let the parse progress */ - def reportTerminalParseMsg(startIndex: Int, - newShortParserMsg: Msgs): Unit = { + def reportTerminalMsg(startIndex: Int, + newShortParserMsg: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex if (!isSuccess && index == traceIndex) failureTerminals :::= newShortParserMsg diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 50023382..a6c213f5 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -82,7 +82,7 @@ trait SharedPackageDefs { val res = if (Util.startsWithIgnoreCase(ctx.input, s, ctx.index)) ctx.freshSuccessUnit(ctx.index + s.length) else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => Util.literalize(s)) + if (ctx.verboseFailures) ctx.reportTerminalMsg(startIndex, () => Util.literalize(s)) res } @@ -106,7 +106,7 @@ trait SharedPackageDefs { else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { ctx.failureGroups = Msgs.empty - ctx.reportTerminalParseMsg(startPos, + ctx.reportTerminalMsg(startPos, if (msg.value.isEmpty) Msgs.empty else () => msg match{ case Seq(x) => s"&(${msg.render})" @@ -127,7 +127,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(startIndex)) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "end-of-input") + if (ctx.verboseFailures) ctx.reportTerminalMsg(startIndex, () => "end-of-input") res } @@ -139,7 +139,7 @@ trait SharedPackageDefs { val res = if (startIndex == 0) ctx.freshSuccessUnit() else ctx.freshFailure().asInstanceOf[P[Unit]] - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "start-of-input") + if (ctx.verboseFailures) ctx.reportTerminalMsg(startIndex, () => "start-of-input") res } @@ -168,7 +168,7 @@ trait SharedPackageDefs { */ def Pass(implicit ctx: P[_]): P[Unit] = { val res = ctx.freshSuccessUnit() - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, Msgs.empty) + if (ctx.verboseFailures) ctx.reportTerminalMsg(ctx.index, Msgs.empty) res } @@ -178,7 +178,7 @@ trait SharedPackageDefs { */ def Pass[T](v: T)(implicit ctx: P[_]): P[T] = { val res = ctx.freshSuccess(v) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Pass") + if (ctx.verboseFailures) ctx.reportTerminalMsg(ctx.index, () => "Pass") res } @@ -187,7 +187,7 @@ trait SharedPackageDefs { */ def Fail(implicit ctx: P[_]): P[Nothing] = { val res = ctx.freshFailure() - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "fail") + if (ctx.verboseFailures) ctx.reportTerminalMsg(ctx.index, () => "fail") res } @@ -199,7 +199,7 @@ trait SharedPackageDefs { */ def Index(implicit ctx: P[_]): P[Int] = { val res = ctx.freshSuccess(ctx.index) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(ctx.index, () => "Index") + if (ctx.verboseFailures) ctx.reportTerminalMsg(ctx.index, () => "Index") res } @@ -212,7 +212,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Unit]] else ctx.freshSuccessUnit(ctx.index + 1) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-char") + if (ctx.verboseFailures) ctx.reportTerminalMsg(startIndex, () => "any-char") res } @@ -228,7 +228,7 @@ trait SharedPackageDefs { val res = if (!ctx.input.isReachable(ctx.index)) ctx.freshFailure().asInstanceOf[P[Char]] else ctx.freshSuccess(ctx.input(ctx.index), ctx.index + 1) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(startIndex, () => "any-char") + if (ctx.verboseFailures) ctx.reportTerminalMsg(startIndex, () => "any-char") res } @@ -263,7 +263,7 @@ object SharedPackageDefs{ if (res.isSuccess) ctx.freshSuccess(ctx.successValue) else ctx.freshFailure(oldIndex) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(oldIndex, () => msg) + if (ctx.verboseFailures) ctx.reportTerminalMsg(oldIndex, () => msg) res2.asInstanceOf[P[T]] } @@ -285,7 +285,7 @@ object SharedPackageDefs{ if (ctx.verboseFailures) { ctx.failureTerminals = startTerminals ctx.failureGroups = Msgs.empty - ctx.reportTerminalParseMsg(startPos, Msgs.empty) + ctx.reportTerminalMsg(startPos, Msgs.empty) } res.cut = startCut res diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 2f41b7ce..f07bc3cb 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -100,7 +100,7 @@ object JavaWhitespace{ else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, () => Util.literalize("*/")) res } } else { @@ -153,7 +153,7 @@ object JsonnetWhitespace{ ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, () => Util.literalize("*/")) res } } else { @@ -205,7 +205,7 @@ object ScalaWhitespace { else { ctx.cut = true val res = ctx.freshFailure(current) - if (ctx.verboseFailures) ctx.reportTerminalParseMsg(current, () => Util.literalize("*/")) + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, () => Util.literalize("*/")) res } } else { From d8bae65b218641cbf3b44f7b0b2a21eba4b88bfa Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 14:15:59 +0800 Subject: [PATCH 30/49] move error reporting into docsite --- .../src-2/fastparse/internal/MacroImpls.scala | 10 +- .../fastparse/internal/MacroRepImpls.scala | 2 +- .../src-2/fastparse/internal/RepImpls.scala | 8 +- .../fastparse/internal/MacroInlineImpls.scala | 10 +- .../fastparse/internal/MacroRepImpls.scala | 2 +- fastparse/src/fastparse/Parsed.scala | 4 +- fastparse/src/fastparse/ParsingRun.scala | 76 ++---------- .../src/fastparse/SharedPackageDefs.scala | 8 +- fastparse/src/fastparse/internal/Util.scala | 2 +- .../test/src/fastparse/ExampleTests.scala | 19 +++ .../test/src/fastparse/FailureTests.scala | 11 +- readme/ErrorReporting.scalatex | 115 ++++++++++++++++++ readme/Readme.scalatex | 2 + 13 files changed, 173 insertions(+), 96 deletions(-) create mode 100644 readme/ErrorReporting.scalatex diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 9a3532f1..33a2af3c 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -231,7 +231,7 @@ object MacroImpls { lhs0.splice val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureGroups + val lhsAggregate = ctx5.failureAggregates if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -250,7 +250,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) + if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -465,7 +465,7 @@ object MacroImpls { else { val $preRhsIndex = $ctx1.index $rhs - val $rhsAggregate = $ctx1.failureGroups + val $rhsAggregate = $ctx1.failureAggregates val $rhsMsg = $ctx1.shortParserMsg val $res = if (!$ctx1.isSuccess) { @@ -491,7 +491,7 @@ object MacroImpls { $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, - // We override the failureGroups to avoid building an `a ~ b` + // We override the failureAggregates to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -521,7 +521,7 @@ object MacroImpls { if (!$ctx1.isSuccess) $ctx1 else { val $postLhsIndex = $ctx1.index - val $lhsAggregate = $ctx1.failureGroups + val $lhsAggregate = $ctx1.failureAggregates val $lhsMsg = $ctx1.shortParserMsg $setCut diff --git a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala index bba8a5fb..9ccbc5bb 100644 --- a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala @@ -75,7 +75,7 @@ object MacroRepImpls{ ${c.prefix}.parse0() val $parsedMsg = $ctx1.shortParserMsg - val $parsedAgg = $ctx1.failureGroups + val $parsedAgg = $ctx1.failureAggregates $originalCut |= $ctx1.cut if (!$ctx1.isSuccess) { val res = diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index f03f27d2..94872a08 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -34,7 +34,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val verboseFailures = ctx.verboseFailures parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroups + val parsedAgg = ctx.failureAggregates val postCut = ctx.cut if (!ctx.isSuccess) { val res = @@ -95,7 +95,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroups + val parsedAgg = ctx.failureAggregates val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -156,7 +156,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ else { parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroups + val parsedAgg = ctx.failureAggregates val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -224,7 +224,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroups + val parsedAgg = ctx.failureAggregates val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess){ diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 87fac304..80de7fac 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -148,7 +148,7 @@ object MacroInlineImpls { if (!ctx1.isSuccess) ctx1 else { val postLhsIndex = ctx1.index - val lhsAggregate = ctx1.failureGroups + val lhsAggregate = ctx1.failureAggregates val lhsMsg = ctx1.shortParserMsg ${ setCut('{ ctx1 }) } @@ -162,7 +162,7 @@ object MacroInlineImpls { else { val preRhsIndex = ctx1.index $rhs - val rhsAggregate = ctx1.failureGroups + val rhsAggregate = ctx1.failureAggregates val rhsMsg = ctx1.shortParserMsg val res = if (!ctx1.isSuccess) { @@ -194,7 +194,7 @@ object MacroInlineImpls { preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, - // We override the failureGroups to avoid building an `a ~ b` + // We override the failureAggregates to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -302,7 +302,7 @@ object MacroInlineImpls { lhs0 val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureGroups + val lhsAggregate = ctx5.failureAggregates if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -321,7 +321,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) { - ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureGroups ::: lhsAggregate) + ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) } ctx5.asInstanceOf[ParsingRun[V]] } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 359a5bce..a80d1598 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -98,7 +98,7 @@ object MacroRepImpls { else { $parse0 val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureGroups + val parsedAgg = ctx.failureAggregates val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index 3b3983ee..a0993f61 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -149,7 +149,7 @@ object Parsed{ assert(!p.isSuccess) TracedFailure( p.failureTerminals, - p.failureGroups, + p.failureAggregates, Parsed.fromParsingRun(p).asInstanceOf[Failure] ) } @@ -216,7 +216,7 @@ object Parsed{ def longTerminalsMsg = Failure.formatMsg(input, stack ++ Seq(terminalAggregateString -> index), index) /** - * A version of [[reportParseMsg]] that includes the parse stack + * A version of [[aggregateMsg]] that includes the parse stack */ def longAggregateMsg = Failure.formatMsg(input, stack ++ Seq(groupAggregateString -> index), index) } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index f70b8302..7a94eb0e 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -109,7 +109,7 @@ final class ParsingRun[+T](val input: ParserInput, val instrument: Instrument, // Mutable vars below: var failureTerminals: Msgs, - var failureGroups: Msgs, + var failureAggregates: Msgs, var shortParserMsg: Msgs, var lastFailureMsg: Msgs, var failureStack: List[(String, Int)], @@ -122,70 +122,6 @@ final class ParsingRun[+T](val input: ParserInput, var noDropBuffer: Boolean, val misc: collection.mutable.Map[Any, Any]){ - // HOW ERROR AGGREGATION WORKS: - // - // Fastparse provides two levels of error aggregation that get enabled when - // calling `.trace()`: `failureTerminals`, and `failureGroups`: - // - // - `failureTerminals` lists all low-level terminal parsers which are - // tried at the given `traceIndex`. This is useful to answer the question - // "what can I put at the error position to make my parse continue" - // - // - `failureGroups` lists all high-level parsers which are tried at - // the given `traceIndex`. This is useful to answer the question "What was - // the parser trying to do when it failed" - // - // The implementation of `failureTerminals` is straightforward: we - // simply call `reportTerminalMsg` in every terminal parser, which collects - // all the messages in a big list and returns it. The implementation of - // `failureGroups` is more interesting, since we need to figure out - // what are the "high level" parsers that we need to list. We use the - // following algorithm: - // - // - When a parse which started at the given `traceIndex` fails without a cut - // - Over-write `failureGroups` with it's `shortParserMsg` - // - // - Otherwise: - // - If we are a terminal parser, we set our `failureGroups` to Nil - // - If we are a compound parser, we simply sum up the `failureGroups` - // of all our constituent parts - // - // The point of this heuristic is to provide the highest-level parsers which - // failed at the `traceIndex`, but are not already part of the `failureStack`. - // non-highest-level parsers do successfully write their message to - // `failureGroups`, but they are subsequently over-written by the higher - // level parsers, until it reaches the point where `cut == true`, indicating - // that any further higher-level parsers will be in `failureStack` and using - // their message to stomp over the existing parse-failure-messages in - // `failureGroups` would be wasteful. - // - // These is an edge case where there is no given failure that occurs exactly at - // `traceIndex` e.g. - // - // - Parsing "ax" with P( ("a" ~ "b") ~ "c" | "a" ~/ "d" ) - // - The final failure `index` and thus `traceIndex` is at offset 1 - // - We would like to receive the aggregation ("b" | "d") - // - But ("a" ~ "b") passes from offsets 0-2, "c" fails at offset 2 and ("a" ~ "b") ~ "c" fails - // from offset 0-2. - // - // In such a case, we truncate the `shortParserMsg` at - // `traceIndex` to only include the portion we're interested in (which directly - // follows the failure). This then gets aggregated nicely to form the error - // message from-point-of-failure. - // - // A follow-on edge case is parsing "ax" with - // - // val inner = P( "a" ~ "b" ) - // P( inner ~ "c" | "a" ~/ "d" ) - // - // - Here, we find that the `inner` parser starts before the `traceIndex` and - // fails at `traceIndex`, - // - But we want our aggregation to continue being ("b" | "d"), rather than - // (inner | "d"). - // - // Thus, for opaque compound parsers like `inner` which do not expose their - // internals, we use `forceAggregate` to force it to expose it's internals - // when it's range covers the `traceIndex` but it isn't an exact match /** * Called by non-terminal parsers after completion, success or failure @@ -207,7 +143,7 @@ final class ParsingRun[+T](val input: ParserInput, def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs): Unit = { - reportParseMsg(startIndex, newShortParserMsg, failureGroups) + reportParseMsg(startIndex, newShortParserMsg, failureAggregates) } def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, @@ -226,7 +162,7 @@ final class ParsingRun[+T](val input: ParserInput, def reportParseMsg(startIndex: Int, newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, failureGroups, forceAggregate, failureGroups.value.nonEmpty) + reportParseMsg0(startIndex, newShortParserMsg, failureAggregates, forceAggregate, failureAggregates.value.nonEmpty) } def reportParseMsg(startIndex: Int, @@ -274,9 +210,9 @@ final class ParsingRun[+T](val input: ParserInput, shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty // There are two cases when aggregating: either we stomp over the entire - // existing `failureGroups` with `newShortParserMsg`, or we preserve it + // existing `failureAggregates` with `newShortParserMsg`, or we preserve it // (with possible additions) with `newFailureGroups`. - failureGroups = + failureAggregates = if (forceAggregate) newFailureGroups else if (discardNewFailureGroups(startIndex)) shortParserMsg else newFailureGroups @@ -351,6 +287,7 @@ final class ParsingRun[+T](val input: ParserInput, def freshFailure(): ParsingRun[Nothing] = { if (verboseFailures){ + println("freshFailure()") lastFailureMsg = null failureStack = Nil } @@ -360,6 +297,7 @@ final class ParsingRun[+T](val input: ParserInput, def freshFailure(startPos: Int): ParsingRun[Nothing] = { if (verboseFailures) { + println(s"freshFailure($startPos)") lastFailureMsg = null failureStack = Nil } diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index a6c213f5..79801087 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -55,7 +55,7 @@ trait SharedPackageDefs { traceIndex = traceIndex, instrument = instrument, failureTerminals = Msgs.empty, - failureGroups = Msgs.empty, + failureAggregates = Msgs.empty, shortParserMsg = Msgs.empty, lastFailureMsg = null, failureStack = List.empty, @@ -105,7 +105,7 @@ trait SharedPackageDefs { if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { - ctx.failureGroups = Msgs.empty + ctx.failureAggregates = Msgs.empty ctx.reportTerminalMsg(startPos, if (msg.value.isEmpty) Msgs.empty else () => msg match{ @@ -157,7 +157,7 @@ trait SharedPackageDefs { val res = p if (ctx.verboseFailures) { - ctx.failureGroups = Msgs.empty + ctx.failureAggregates = Msgs.empty ctx.shortParserMsg = Msgs.empty } res @@ -284,7 +284,7 @@ object SharedPackageDefs{ if (ctx.verboseFailures) { ctx.failureTerminals = startTerminals - ctx.failureGroups = Msgs.empty + ctx.failureAggregates = Msgs.empty ctx.reportTerminalMsg(startPos, Msgs.empty) } res.cut = startCut diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 2d0c1250..abcb1e65 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -122,7 +122,7 @@ object Util { // we backtrack past the sep on failure) as well as the failure // aggregate of the previous rep, which we could have continued val newAgg = - if (sepMsg == null || precut) ctx.failureGroups + if (sepMsg == null || precut) ctx.failureAggregates else Util.joinBinOp(sepMsg, parsedMsg) ctx.reportParseMsg( diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index 05ca749a..46dc7f7e 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -551,5 +551,24 @@ object ExampleTests extends TestSuite{ check("oR", "Parsed: Or") check("IllegalBooleanOperation", "Cannot parse IllegalBooleanOperation as an AndOr") } + test("errorHandlingExplanation") { + import fastparse._, NoWhitespace._ + def num[$: P] = P(CharIn("0-9")).log + def sum[$: P] = P("(" ~/ expr ~ "+" ~/ expr ~ ")").log + def expr[$: P]: P[_] = P(num | sum).log + + val Parsed.Failure(_, _, extra) = fastparse.parse("(1?2)", expr(_)) + val trace = extra.trace() + assert( + trace.longTerminalsMsg == + """Expected expr:1:1 / sum:1:1 / expr:1:4 / ([0-9] | "("):1:4, found "?"""" + ) + assert( + trace.longAggregateMsg == + """Expected expr:1:1 / sum:1:1 / expr:1:4 / (num | sum):1:4, found "?"""" + ) + + } } + } diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index cda4ba68..ebd1ee38 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -31,16 +31,19 @@ object FailureTests extends TestSuite{ val f @ Parsed.Failure(failureString, index, extra) = parse("d", parser(_)) val trace = f.trace(true) + val groupAggregateString = trace.groupAggregateString + println("trace.longMsg " + trace.longAggregateMsg) assert( - trace.terminalAggregateString == """("a" | "b" | "c")""", - trace.groupAggregateString == """(parseB | "c")""" + groupAggregateString == """(parseB | "c")""" +// trace.terminalAggregateString == """("a" | "b" | "c" | "x")""" ) } test("either") - check{ def parseB[$: P] = P( "a" | "b" ) def parseA[$: P] = P( (parseB | "") ~ "c" ) - parseA(_) + def parse0[$: P] = P( ("" ~/ parseA).? ~ "x" ) + parse0(_) } test("option") - check{ def parseB[$: P] = P( "a" | "b" ) @@ -436,7 +439,7 @@ object FailureTests extends TestSuite{ import NoWhitespace._ // In the case where one branch fails further in than `traceIndex`, we // collect the partial aggregation from that branch in the - // `failureGroups` but ignore that branch's downstream failure in + // `failureAggregates` but ignore that branch's downstream failure in // `failureTerminalsAggregate` def check(parser: P[_] => P[_]) = checkOffset( diff --git a/readme/ErrorReporting.scalatex b/readme/ErrorReporting.scalatex new file mode 100644 index 00000000..bc1a0139 --- /dev/null +++ b/readme/ErrorReporting.scalatex @@ -0,0 +1,115 @@ +@import Main._ +@sect{Error Reporting} + @p + This section goes into detail of how the FastParse error reporting + algorithm works. In general, it should "just work" when you call + @code{.longMsg}, @code{.longAggregateMsg}, or @code{.longTerminalsMsg}. + Nevertheless, it is both complicated as well important enough that it + is worth documenting in detail + + @p + Fastparse provides two levels of error reporting that get enabled when + calling @code{.trace()}. For the following explanations, consider an example + failed parse of a simplified arithmetic parser: + + @hl.ref(tests/"ExampleTests.scala", Seq("\"errorHandlingExplanation\"", "")) + + @p + This fails on the @code{?} being invalid syntax. The following error reporting + levels will treat this as follows: + @ul + @li + @code{failureTerminals} lists all the lowest-level terminal parsers which are + tried at the given @code{traceIndex}, i.e. the character class @code{[0-9]} and the + token "(". This is useful to answer the question "what token/char can I + put at the error position to make my parse continue". The implementation + of @code{failureTerminals} is straightforward: we simply call + @code{reportTerminalMsg} in every terminal parser, which collects all the + messages in a big list and returns it. + @li + @code{failureAggregates} lists all high-level parsers which are tried at the given + @code{traceIndex}, i.e. the named parsers @code{num} and @code{plus}. This is useful to + answer the question "What construct was the parser trying to do when it + failed" + @p + The implementation of @code{failureAggregates} is more interesting, since we need + to define what "high level" parsers means, which is non-obvious. Fastparse + uses the following definition: + @ul + @li + @code{failureAggregates} should contain the parsers highest in the call stack, + whose failure isn't immediately fatal to the parse (due to them being in + @code{|}, @code{.rep}, @code{?}, or other "backtrackable" operators, but + not past a @code{cut}) + @p + This is a useful definition because we already have the @code{failureStack} + containing all (named) parsers whose failure *is* immediately fatal to the + parse, both those at @code{traceIndex} and those earlier in the input. This + there is no need to duplicate showing any of them in the @code{failureAggregates}, + and we can instead go "one level deeper" to find the highest-level parsers + within the deepest parser of the @code{failureStack} and show those instead. + Thus, in the combined @code{longAggregateMsg}, the failure stack shows us + exactly which parsers failing directly contributed to the failure at + @code{traceIndex}, while the longAggregateMsg tells us what are the + highest-level parsers FastParse was trying to parse at @code{traceIndex} before + it finally failed. + @p + To collect the @code{failureAggregates}, We use the following algorithm: + @ul + @li + When a parse which started at the given @code{traceIndex} fails without a cut: + Over-write @code{failureAggregates} with it's @code{shortParserMsg} + + @li + Otherwise: + + @ul + @li + If we are a terminal parser, we set our @code{failureAggregates} to Nil + @li + If we are a compound parser, we simply sum up the @code{failureAggregates} + of all our constituent parts + @p + The point of this heuristic is to provide the highest-level parsers which + failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. + non-highest-level parsers do successfully write their message to + @code{failureAggregates}, but they are subsequently over-written by the higher + level parsers, until it reaches the point where @code{cut == true}, indicating + that any further higher-level parsers will be in @code{failureStack} and using + their message to stomp over the existing parse-failure-messages in + @code{failureAggregates} would be wasteful. + @p + These is an edge case where there is no given failure that occurs exactly at + @code{traceIndex} e.g. + @ul + @li + Parsing @hl.scala{"ax"} with @hl.scala{P( ("a" ~ "b") ~ "c" | "a" ~/ "d" )} + @li + The final failure @code{index} and thus @code{traceIndex} is at offset 1 + @li + We would like to receive the aggregation @hl.scala{("b" | "d")} + @li + But @hl.scala{("a" ~ "b")} passes from offsets 0-2, @hl.scala{"c"} fails + + + @p + In such a case, we truncate the @code{shortParserMsg} at + @code{traceIndex} to only include the portion we're interested in (which directly + follows the failure). This then gets aggregated nicely to form the error + message from-point-of-failure. + @p + A follow-on edge case is parsing "ax" with + @hl.scala + val inner = P( "a" ~ "b" ) + P( inner ~ "c" | "a" ~/ "d" ) + @ul + @li + Here, we find that the @code{inner} parser starts before the @code{traceIndex} and + fails at @code{traceIndex}, + @li + But we want our aggregation to continue being @hl.scala{("b" | "d")}, rather than + @hl.scala{(inner | "d")}. + + Thus, for opaque compound parsers like @code{inner} which do not expose their + internals, we use @code{forceAggregate} to force it to expose it's internals + when it's range covers the @code{traceIndex} but it isn't an exact match diff --git a/readme/Readme.scalatex b/readme/Readme.scalatex index 1e2951c3..0e87d746 100644 --- a/readme/Readme.scalatex +++ b/readme/Readme.scalatex @@ -39,5 +39,7 @@ @FastParseInternals() + @ErrorReporting() + @Changelog() From 1be49e161ba0b814c30610acd17dbf1d8fe1d0f3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 14:58:26 +0800 Subject: [PATCH 31/49] . --- build.sbt | 2 +- project/build.properties | 2 +- ....scalatex => ErrorReportingInternals.scalatex} | 15 ++++++++------- readme/ExampleParsers.scalatex | 3 ++- readme/FastParseInternals.scalatex | 2 +- readme/Readme.scalatex | 4 ++-- readme/StreamingParsing.scalatex | 3 ++- 7 files changed, 17 insertions(+), 14 deletions(-) rename readme/{ErrorReporting.scalatex => ErrorReportingInternals.scalatex} (91%) diff --git a/build.sbt b/build.sbt index 77002b9f..968e5133 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ lazy val readme = scalatex.ScalatexReadme( source = "Readme", autoResources = List("out.js", "JProfiler.png") ).settings( - (resources in Compile) += baseDirectory.value/".."/"out"/"demo"/"fullOpt"/"dest"/"out.js", + (resources in Compile) += baseDirectory.value/".."/"out"/"demo"/"fullOpt.dest"/"out.js", scalaVersion := "2.12.10" ) diff --git a/project/build.properties b/project/build.properties index 7c58a83a..46e43a97 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.6 +sbt.version=1.8.2 diff --git a/readme/ErrorReporting.scalatex b/readme/ErrorReportingInternals.scalatex similarity index 91% rename from readme/ErrorReporting.scalatex rename to readme/ErrorReportingInternals.scalatex index bc1a0139..697caf1d 100644 --- a/readme/ErrorReporting.scalatex +++ b/readme/ErrorReportingInternals.scalatex @@ -1,5 +1,6 @@ @import Main._ -@sect{Error Reporting} +@val tests = wd/'fastparse/'test/'src/'fastparse +@sect{Error Reporting Internals} @p This section goes into detail of how the FastParse error reporting algorithm works. In general, it should "just work" when you call @@ -8,9 +9,9 @@ is worth documenting in detail @p - Fastparse provides two levels of error reporting that get enabled when - calling @code{.trace()}. For the following explanations, consider an example - failed parse of a simplified arithmetic parser: + The two levels of error reporting that are most interesting are + @code{.longAggregateMsg} and @code{.longTerminalsMsg}. Consider a failed + parse of an example simplified arithmetic parser: @hl.ref(tests/"ExampleTests.scala", Seq("\"errorHandlingExplanation\"", "")) @@ -21,7 +22,7 @@ @li @code{failureTerminals} lists all the lowest-level terminal parsers which are tried at the given @code{traceIndex}, i.e. the character class @code{[0-9]} and the - token "(". This is useful to answer the question "what token/char can I + token @hl.scala{"("}. This is useful to answer the question "what token/char can I put at the error position to make my parse continue". The implementation of @code{failureTerminals} is straightforward: we simply call @code{reportTerminalMsg} in every terminal parser, which collects all the @@ -44,7 +45,7 @@ @p This is a useful definition because we already have the @code{failureStack} containing all (named) parsers whose failure *is* immediately fatal to the - parse, both those at @code{traceIndex} and those earlier in the input. This + parse, both those at @code{traceIndex} and those earlier in the input. Thus there is no need to duplicate showing any of them in the @code{failureAggregates}, and we can instead go "one level deeper" to find the highest-level parsers within the deepest parser of the @code{failureStack} and show those instead. @@ -70,7 +71,7 @@ If we are a compound parser, we simply sum up the @code{failureAggregates} of all our constituent parts @p - The point of this heuristic is to provide the highest-level parsers which + As mentioned earlier, the point of this is to provide the highest-level parsers which failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. non-highest-level parsers do successfully write their message to @code{failureAggregates}, but they are subsequently over-written by the higher diff --git a/readme/ExampleParsers.scalatex b/readme/ExampleParsers.scalatex index ea9e3865..79093ddd 100644 --- a/readme/ExampleParsers.scalatex +++ b/readme/ExampleParsers.scalatex @@ -1,5 +1,6 @@ @import Main._ @val tests = wd/'fastparse/'test/'src/'fastparse +@val tests212plus = wd/'fastparse/'test/"src-2.12+"/'fastparse @val main = wd/'fastparse/'src/'fastparse @sect{Example Parsers} @@ -76,7 +77,7 @@ You can also define your own custom whitespace consumer, if none of bundled ones fit your needs: - @hl.ref(tests/"CustomWhitespaceMathTests.scala", "implicit val whitespace", "val tests") + @hl.ref(tests212plus/"CustomWhitespaceMathTests.scala", "implicit object whitespace", "val tests") diff --git a/readme/FastParseInternals.scalatex b/readme/FastParseInternals.scalatex index 8e5d03d0..162a4ab2 100644 --- a/readme/FastParseInternals.scalatex +++ b/readme/FastParseInternals.scalatex @@ -1,7 +1,7 @@ @import Main._ @sect{Internals} @p - FastParse 2.0.5 is implemented as a set of methods that perform a + FastParse is implemented as a set of methods that perform a recursive-descent parse on the given input, with all book-keeping information maintained in the @code{fastparse.ParsingRun[T]} objects (abbreviated @code{fastparse.P[T]}). @code{ParsingRun}s are mutable, diff --git a/readme/Readme.scalatex b/readme/Readme.scalatex index 0e87d746..4459d7a1 100644 --- a/readme/Readme.scalatex +++ b/readme/Readme.scalatex @@ -20,7 +20,7 @@ ) ) -@sect("FastParse 2.2.2", "Fast to write, Fast running Parsers in Scala") +@sect("FastParse 3.0.0", "Fast to write, Fast running Parsers in Scala") @GettingStarted() @WritingParsers() @@ -39,7 +39,7 @@ @FastParseInternals() - @ErrorReporting() + @ErrorReportingInternals() @Changelog() diff --git a/readme/StreamingParsing.scalatex b/readme/StreamingParsing.scalatex index c08d7fdc..09c0ff65 100644 --- a/readme/StreamingParsing.scalatex +++ b/readme/StreamingParsing.scalatex @@ -1,5 +1,6 @@ @import Main._ @val tests = wd/'fastparse/'test/'src/'fastparse +@val tests212plus = wd/'fastparse/'test/"src-2.12+"/'fastparse @val main = wd/'fastparse/'src/'fastparse @sect{Streaming Parsing} @@ -9,7 +10,7 @@ @hl.scala{Iterator[String]} or @hl.scala{java.io.InputStream} instead of a @code{String} to the @hl.scala{fastparse.parse} method. - @hl.ref(tests/"IteratorTests.scala", Seq("\"basic\"", "")) + @hl.ref(tests212plus/"IteratorTests.scala", Seq("\"basic\"", "")) @p Streaming parsing still needs to buffer input in-memory: in particular, From 883e5e7bbfc44ee2e606e214a3547a8d2494192f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:02:08 +0800 Subject: [PATCH 32/49] . --- fastparse/src/fastparse/ParsingRun.scala | 2 - .../test/src/fastparse/ExampleTests.scala | 9 +- readme/ErrorReportingInternals.scalatex | 155 +++++++++--------- 3 files changed, 85 insertions(+), 81 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 7a94eb0e..2bb3a9d7 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -287,7 +287,6 @@ final class ParsingRun[+T](val input: ParserInput, def freshFailure(): ParsingRun[Nothing] = { if (verboseFailures){ - println("freshFailure()") lastFailureMsg = null failureStack = Nil } @@ -297,7 +296,6 @@ final class ParsingRun[+T](val input: ParserInput, def freshFailure(startPos: Int): ParsingRun[Nothing] = { if (verboseFailures) { - println(s"freshFailure($startPos)") lastFailureMsg = null failureStack = Nil } diff --git a/fastparse/test/src/fastparse/ExampleTests.scala b/fastparse/test/src/fastparse/ExampleTests.scala index 46dc7f7e..6e6b0108 100644 --- a/fastparse/test/src/fastparse/ExampleTests.scala +++ b/fastparse/test/src/fastparse/ExampleTests.scala @@ -557,15 +557,16 @@ object ExampleTests extends TestSuite{ def sum[$: P] = P("(" ~/ expr ~ "+" ~/ expr ~ ")").log def expr[$: P]: P[_] = P(num | sum).log - val Parsed.Failure(_, _, extra) = fastparse.parse("(1?2)", expr(_)) + val Parsed.Failure(_, _, extra) = fastparse.parse("(1+?)", expr(_)) val trace = extra.trace() + val longTerminalsMsg = trace.longTerminalsMsg assert( - trace.longTerminalsMsg == - """Expected expr:1:1 / sum:1:1 / expr:1:4 / ([0-9] | "("):1:4, found "?"""" + longTerminalsMsg == + """Expected expr:1:1 / sum:1:1 / expr:1:4 / ([0-9] | "("):1:4, found "?)"""" ) assert( trace.longAggregateMsg == - """Expected expr:1:1 / sum:1:1 / expr:1:4 / (num | sum):1:4, found "?"""" + """Expected expr:1:1 / sum:1:1 / expr:1:4 / (num | sum):1:4, found "?)"""" ) } diff --git a/readme/ErrorReportingInternals.scalatex b/readme/ErrorReportingInternals.scalatex index 697caf1d..1552b0ad 100644 --- a/readme/ErrorReportingInternals.scalatex +++ b/readme/ErrorReportingInternals.scalatex @@ -34,83 +34,88 @@ failed" @p The implementation of @code{failureAggregates} is more interesting, since we need - to define what "high level" parsers means, which is non-obvious. Fastparse - uses the following definition: - @ul - @li - @code{failureAggregates} should contain the parsers highest in the call stack, - whose failure isn't immediately fatal to the parse (due to them being in - @code{|}, @code{.rep}, @code{?}, or other "backtrackable" operators, but - not past a @code{cut}) - @p - This is a useful definition because we already have the @code{failureStack} - containing all (named) parsers whose failure *is* immediately fatal to the - parse, both those at @code{traceIndex} and those earlier in the input. Thus - there is no need to duplicate showing any of them in the @code{failureAggregates}, - and we can instead go "one level deeper" to find the highest-level parsers - within the deepest parser of the @code{failureStack} and show those instead. - Thus, in the combined @code{longAggregateMsg}, the failure stack shows us - exactly which parsers failing directly contributed to the failure at - @code{traceIndex}, while the longAggregateMsg tells us what are the - highest-level parsers FastParse was trying to parse at @code{traceIndex} before - it finally failed. - @p - To collect the @code{failureAggregates}, We use the following algorithm: - @ul - @li - When a parse which started at the given @code{traceIndex} fails without a cut: - Over-write @code{failureAggregates} with it's @code{shortParserMsg} + to define what "high level" parsers means, which is non-obvious. - @li - Otherwise: + @sect{Definition of failureAggregates} + @p + Fastparse uses the following definition for @code{failureAggregates}: + @ul + @li + @code{failureAggregates} should contain the parsers highest in the call stack, + whose failure isn't immediately fatal to the parse (due to them being in + @code{|}, @code{.rep}, @code{?}, or other "backtrackable" operators, but + not past a @code{cut}) + @p + This is a useful definition because we already have the @code{failureStack} + containing all (named) parsers whose failure *is* immediately fatal to the + parse, both those at @code{traceIndex} and those earlier in the input. Thus + there is no need to duplicate showing any of them in the @code{failureAggregates}, + and we can instead go "one level deeper" to find the highest-level parsers + within the deepest parser of the @code{failureStack} and show those instead. + Thus, in the combined @code{longAggregateMsg}, the failure stack shows us + exactly which parsers failing directly contributed to the failure at + @code{traceIndex}, while the longAggregateMsg tells us what are the + highest-level parsers FastParse was trying to parse at @code{traceIndex} before + it finally failed. + @sect{Implementation of failureAggregates} + @p + To collect the @code{failureAggregates}, We use the following algorithm: + @ul + @li + When a parse which started at the given @code{traceIndex} fails without a cut: + Over-write @code{failureAggregates} with it's @code{shortParserMsg} - @ul - @li - If we are a terminal parser, we set our @code{failureAggregates} to Nil - @li - If we are a compound parser, we simply sum up the @code{failureAggregates} - of all our constituent parts - @p - As mentioned earlier, the point of this is to provide the highest-level parsers which - failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. - non-highest-level parsers do successfully write their message to - @code{failureAggregates}, but they are subsequently over-written by the higher - level parsers, until it reaches the point where @code{cut == true}, indicating - that any further higher-level parsers will be in @code{failureStack} and using - their message to stomp over the existing parse-failure-messages in - @code{failureAggregates} would be wasteful. - @p - These is an edge case where there is no given failure that occurs exactly at - @code{traceIndex} e.g. - @ul - @li - Parsing @hl.scala{"ax"} with @hl.scala{P( ("a" ~ "b") ~ "c" | "a" ~/ "d" )} - @li - The final failure @code{index} and thus @code{traceIndex} is at offset 1 - @li - We would like to receive the aggregation @hl.scala{("b" | "d")} - @li - But @hl.scala{("a" ~ "b")} passes from offsets 0-2, @hl.scala{"c"} fails + @li + Otherwise: + @ul + @li + If we are a terminal parser, we set our @code{failureAggregates} to Nil + @li + If we are a compound parser, we simply sum up the @code{failureAggregates} + of all our constituent parts + @p + As mentioned earlier, the point of this is to provide the highest-level parsers which + failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. + non-highest-level parsers do successfully write their message to + @code{failureAggregates}, but they are subsequently over-written by the higher + level parsers, until it reaches the point where @code{cut == true}, indicating + that any further higher-level parsers will be in @code{failureStack} and using + their message to stomp over the existing parse-failure-messages in + @code{failureAggregates} would be wasteful. + @sect{Edge Cases} + @p + These is an edge case where there is no given failure that occurs exactly at + @code{traceIndex} e.g. + @ul + @li + Parsing @hl.scala{"ax"} with @hl.scala{P( ("a" ~ "b") ~ "c" | "a" ~/ "d" )} + @li + The final failure @code{index} and thus @code{traceIndex} is at offset 1 + @li + We would like to receive the aggregation @hl.scala{("b" | "d")} + @li + But @hl.scala{("a" ~ "b")} passes from offsets 0-2, @hl.scala{"c"} fails - @p - In such a case, we truncate the @code{shortParserMsg} at - @code{traceIndex} to only include the portion we're interested in (which directly - follows the failure). This then gets aggregated nicely to form the error - message from-point-of-failure. - @p - A follow-on edge case is parsing "ax" with - @hl.scala - val inner = P( "a" ~ "b" ) - P( inner ~ "c" | "a" ~/ "d" ) - @ul - @li - Here, we find that the @code{inner} parser starts before the @code{traceIndex} and - fails at @code{traceIndex}, - @li - But we want our aggregation to continue being @hl.scala{("b" | "d")}, rather than - @hl.scala{(inner | "d")}. - Thus, for opaque compound parsers like @code{inner} which do not expose their - internals, we use @code{forceAggregate} to force it to expose it's internals - when it's range covers the @code{traceIndex} but it isn't an exact match + @p + In such a case, we truncate the @code{shortParserMsg} at + @code{traceIndex} to only include the portion we're interested in (which directly + follows the failure). This then gets aggregated nicely to form the error + message from-point-of-failure. + @p + A follow-on edge case is parsing "ax" with + @hl.scala + val inner = P( "a" ~ "b" ) + P( inner ~ "c" | "a" ~/ "d" ) + @ul + @li + Here, we find that the @code{inner} parser starts before the @code{traceIndex} and + fails at @code{traceIndex}, + @li + But we want our aggregation to continue being @hl.scala{("b" | "d")}, rather than + @hl.scala{(inner | "d")}. + + Thus, for opaque compound parsers like @code{inner} which do not expose their + internals, we use @code{forceAggregate} to force it to expose it's internals + when it's range covers the @code{traceIndex} but it isn't an exact match From 1a5820d5d7a3196236ca3a90e22369c946bfcc13 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:08:38 +0800 Subject: [PATCH 33/49] rename callbacks again --- .../src-2/fastparse/internal/MacroImpls.scala | 10 +++---- .../fastparse/internal/MacroInlineImpls.scala | 10 +++---- fastparse/src/fastparse/ParsingRun.scala | 28 +++++++++---------- fastparse/src/fastparse/internal/Util.scala | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 33a2af3c..2a7a3ede 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -52,7 +52,7 @@ object MacroImpls { ctx1.instrument.afterParse(name.splice.value, ctx0.index, ctx0.isSuccess) } if (ctx0.verboseFailures) { - ctx0.reportParseMsg( + ctx0.reportAggregateMsg( startIndex, () => name.splice.value, forceAggregate = startIndex < ctx0.traceIndex @@ -241,7 +241,7 @@ object MacroImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg) + if (verboseFailures) ctx5.reportAggregateMsg(startPos, lhsMsg) ctx5.cut = false other.splice @@ -250,7 +250,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) + if (verboseFailures) ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -487,7 +487,7 @@ object MacroImpls { ) } - if ($ctx1.verboseFailures) $ctx1.reportParseMsg( + if ($ctx1.verboseFailures) $ctx1.reportAggregateMsg( $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, @@ -693,7 +693,7 @@ object MacroImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg if (!postSuccess){ - ctx1.reportParseMsg(startPos, () => msg.render + ".?") + ctx1.reportAggregateMsg(startPos, () => msg.render + ".?") } } res diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 80de7fac..7cdc0355 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -103,7 +103,7 @@ object MacroInlineImpls { ctx1.instrument.afterParse(name.value, ctx0.index, ctx0.isSuccess) } if (ctx0.verboseFailures) { - ctx0.reportParseMsg( + ctx0.reportAggregateMsg( startIndex, () => name.value, forceAggregate = startIndex < ctx0.traceIndex @@ -190,7 +190,7 @@ object MacroInlineImpls { if (ctx1.verboseFailures) { // println("rhsAggregate " + rhsAggregate) // println("lhsAggregate " + lhsAggregate) - ctx1.reportParseMsg( + ctx1.reportAggregateMsg( preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, @@ -248,7 +248,7 @@ object MacroInlineImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg if (!postSuccess) { - ctx1.reportParseMsg(startPos, () => msg.render + ".?") + ctx1.reportAggregateMsg(startPos, () => msg.render + ".?") } } res @@ -311,7 +311,7 @@ object MacroInlineImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportParseMsg(startPos, lhsMsg) + if (verboseFailures) ctx5.reportAggregateMsg(startPos, lhsMsg) ctx5.cut = false other @@ -321,7 +321,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) { - ctx5.reportParseMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) + ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) } ctx5.asInstanceOf[ParsingRun[V]] } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 2bb3a9d7..cc0409d2 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -140,16 +140,16 @@ final class ParsingRun[+T](val input: ParserInput, * of inlining in Fastparse, and large amounts of bytecode inlined in a method * can cause JVM performance problems (e.g. JIT compilation may get disabled) */ - def reportParseMsg(startIndex: Int, - newShortParserMsg: Msgs): Unit = { + def reportAggregateMsg(startIndex: Int, + newShortParserMsg: Msgs): Unit = { - reportParseMsg(startIndex, newShortParserMsg, failureAggregates) + reportAggregateMsg(startIndex, newShortParserMsg, failureAggregates) } - def reportParseMsg(startIndex: Int, - newShortParserMsg: Msgs, - newFailureGroups: Msgs): Unit = { + def reportAggregateMsg(startIndex: Int, + newShortParserMsg: Msgs, + newFailureGroups: Msgs): Unit = { - reportParseMsg(startIndex, newShortParserMsg, newFailureGroups, false) + reportAggregateMsg(startIndex, newShortParserMsg, newFailureGroups, false) } /** @@ -159,16 +159,16 @@ final class ParsingRun[+T](val input: ParserInput, * parsers that have either succeeded past the traceIndex, or failed and * potentially backtracked. */ - def reportParseMsg(startIndex: Int, - newShortParserMsg: Msgs, - forceAggregate: Boolean): Unit = { + def reportAggregateMsg(startIndex: Int, + newShortParserMsg: Msgs, + forceAggregate: Boolean): Unit = { reportParseMsg0(startIndex, newShortParserMsg, failureAggregates, forceAggregate, failureAggregates.value.nonEmpty) } - def reportParseMsg(startIndex: Int, - newShortParserMsg: Msgs, - newFailureGroups: Msgs, - forceAggregate: Boolean): Unit = { + def reportAggregateMsg(startIndex: Int, + newShortParserMsg: Msgs, + newFailureGroups: Msgs, + forceAggregate: Boolean): Unit = { reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) } diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index abcb1e65..d2b94128 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -125,7 +125,7 @@ object Util { if (sepMsg == null || precut) ctx.failureAggregates else Util.joinBinOp(sepMsg, parsedMsg) - ctx.reportParseMsg( + ctx.reportAggregateMsg( startIndex, () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), if (lastAgg == null) newAgg From 97f3ff8196dc1fd26b9b09baaafc72c998e9eb21 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:11:56 +0800 Subject: [PATCH 34/49] simplify --- fastparse/src/fastparse/ParsingRun.scala | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index cc0409d2..efa313ce 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -152,24 +152,29 @@ final class ParsingRun[+T](val input: ParserInput, reportAggregateMsg(startIndex, newShortParserMsg, newFailureGroups, false) } - /** - * We only want to set the shortMsg for most parsers if they could have - * potentially extended passed the [[traceIndex]], since that is the point at - * which all error reporting in Fastparse is focused. That means we want - * parsers that have either succeeded past the traceIndex, or failed and - * potentially backtracked. - */ def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, failureAggregates, forceAggregate, failureAggregates.value.nonEmpty) + reportAggregateMsg(startIndex, newShortParserMsg, failureAggregates, forceAggregate) } def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, newFailureGroups: Msgs, forceAggregate: Boolean): Unit = { - reportParseMsg0(startIndex, newShortParserMsg, newFailureGroups, forceAggregate, true) + + reportParseMsg0( + startIndex, + newShortParserMsg, + newFailureGroups, + forceAggregate, + // We only want to set the shortMsg for most parsers if they could have + // potentially extended passed the [[traceIndex]], since that is the point at + // which all error reporting in Fastparse is focused. That means we want + // parsers that have either succeeded past the traceIndex, or failed and + // potentially backtracked. + newFailureGroups.value.nonEmpty + ) } /** From 87c77431f839da534c4f093df63f470fcd75908f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:13:43 +0800 Subject: [PATCH 35/49] cleanup --- fastparse/test/src/fastparse/FailureTests.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index ebd1ee38..f7268ac9 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -31,11 +31,9 @@ object FailureTests extends TestSuite{ val f @ Parsed.Failure(failureString, index, extra) = parse("d", parser(_)) val trace = f.trace(true) - val groupAggregateString = trace.groupAggregateString - println("trace.longMsg " + trace.longAggregateMsg) assert( - groupAggregateString == """(parseB | "c")""" -// trace.terminalAggregateString == """("a" | "b" | "c" | "x")""" + trace.groupAggregateString == """(parseB | "c")""", + trace.terminalAggregateString == """("a" | "b" | "c" | "x")""" ) } From 82b7bf0cf1e16ffbcecee84223e7abc8bab6e44f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:14:49 +0800 Subject: [PATCH 36/49] cleanup --- .../fastparse/internal/MacroInlineImpls.scala | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 7cdc0355..ee80c7dc 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -187,20 +187,16 @@ object MacroInlineImpls { ) } - if (ctx1.verboseFailures) { -// println("rhsAggregate " + rhsAggregate) -// println("lhsAggregate " + lhsAggregate) - ctx1.reportAggregateMsg( - preLhsIndex, - Util.joinBinOp(lhsMsg, rhsMsg), - rhsAggregate ::: lhsAggregate, - // We override the failureAggregates to avoid building an `a ~ b` - // aggregate msg in the specific case where the LHS parser fails to - // make any progress past `startIndex`. This finds cases like `a.? ~ b` - // or `a.rep ~ b` and lets use flatten them out into `a | b` - forceAggregate = preRhsIndex == ctx1.traceIndex - ) - } + if (ctx1.verboseFailures) ctx1.reportAggregateMsg( + preLhsIndex, + Util.joinBinOp(lhsMsg, rhsMsg), + rhsAggregate ::: lhsAggregate, + // We override the failureAggregates to avoid building an `a ~ b` + // aggregate msg in the specific case where the LHS parser fails to + // make any progress past `startIndex`. This finds cases like `a.? ~ b` + // or `a.rep ~ b` and lets use flatten them out into `a | b` + forceAggregate = preRhsIndex == ctx1.traceIndex + ) res } } From 1ad14fc259bc4a27f67cf316d0f4c772cf21d0ab Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:15:24 +0800 Subject: [PATCH 37/49] cleanup --- scalaparse/test/src/scalaparse/TestUtil.scala | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/scalaparse/test/src/scalaparse/TestUtil.scala b/scalaparse/test/src/scalaparse/TestUtil.scala index ae2fd9dc..7fdc7349 100644 --- a/scalaparse/test/src/scalaparse/TestUtil.scala +++ b/scalaparse/test/src/scalaparse/TestUtil.scala @@ -27,27 +27,16 @@ object TestUtil { val parsedFound = input.slice(f.index, f.index + 10) val stack = trace.longAggregateMsg - assert({ - implicitly(input) - implicitly(stack) - implicitly(index) - implicitly(parsedFound) - (aggregate == null || aggregate.trim == parsedAggregate.trim) - }) - assert({ - implicitly(input) - implicitly(stack) - implicitly(index) - implicitly(parsedFound) - (terminals == null || terminals.trim == parsedTerminals.trim) - }) - assert({ - implicitly(input) + assert( + { implicitly(input) implicitly(stack) implicitly(index) implicitly(parsedFound) + (aggregate == null || aggregate.trim == parsedAggregate.trim) && + (terminals == null || terminals.trim == parsedTerminals.trim) && parsedFound.startsWith(found) - }) + } + ) line.value case _: Parsed.Success[_] => From 793f6849dbd8ab68c8eb1368b474db2221010a39 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 15:15:53 +0800 Subject: [PATCH 38/49] cleanup --- scalaparse/src/scalaparse/syntax/Literals.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scalaparse/src/scalaparse/syntax/Literals.scala b/scalaparse/src/scalaparse/syntax/Literals.scala index f6366747..14e8985f 100644 --- a/scalaparse/src/scalaparse/syntax/Literals.scala +++ b/scalaparse/src/scalaparse/syntax/Literals.scala @@ -58,7 +58,7 @@ trait Literals { l => def Null[$: P] = Key.W("null") def OctalEscape[$: P] = P( Digit ~ Digit.? ~ Digit.? ) - def Escape[$: P] = P( "\\" ~/ (CharIn("""btnfr'\\"]""") | OctalEscape | UnicodeEscape ) ).log + def Escape[$: P] = P( "\\" ~/ (CharIn("""btnfr'\\"]""") | OctalEscape | UnicodeEscape ) ) // Note that symbols can take on the same values as keywords! def Symbol[$: P] = P( Identifiers.PlainId | Identifiers.Keywords ) @@ -93,7 +93,6 @@ trait Literals { l => def NonStringEnd = P( !CharIn("\n\"") ~ AnyChar ) P( (StringChars | Interp | LiteralSlash | Escape | NonStringEnd ).rep ) } - def String[$: P] = { P { Id.filter(_ => interp.isDefined) ~ ( @@ -104,10 +103,11 @@ trait Literals { l => "\"" ~/ NoInterp.SingleChars(false) ~ "\"" } } - } + } def NoInterp[$: P] = new InterpCtx(None) def Pat[$: P] = new InterpCtx(Some(() => l.Pattern)) def Expr[$: P] = new InterpCtx(Some(() => Block)) + } } From 645be88a068b509379f3e5e3428fb85d57f33548 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:24:12 +0800 Subject: [PATCH 39/49] cleanup --- fastparse/test/src/fastparse/FailureTests.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index f7268ac9..d12093c5 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -12,16 +12,14 @@ object FailureTests extends TestSuite{ terminals: String = null, parser: P[_] => P[_]) = { val f @ Parsed.Failure(failureString, index, extra) = parse(input, parser(_)) + val trace = f.trace(true) val terminals1 = Option(terminals).getOrElse(expected) - val groupAggregateString = trace.groupAggregateString - val traceLabel = trace.failure.label - val traceTerminalAggregateString = trace.terminalAggregateString assert( - groupAggregateString == expected, - traceLabel == label, - traceTerminalAggregateString == terminals1 + trace.groupAggregateString == expected, + trace.label == label, + trace.terminalAggregateString == terminals1 ) } From 2a1f12e840af9cd6fadc3c8983df0e063aa928c8 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:30:02 +0800 Subject: [PATCH 40/49] cleanup --- fastparse/src/fastparse/ParsingRun.scala | 18 ++++++++++++++---- .../test/src/fastparse/FailureTests.scala | 5 ++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index efa313ce..d10d7962 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -169,10 +169,20 @@ final class ParsingRun[+T](val input: ParserInput, newFailureGroups, forceAggregate, // We only want to set the shortMsg for most parsers if they could have - // potentially extended passed the [[traceIndex]], since that is the point at - // which all error reporting in Fastparse is focused. That means we want - // parsers that have either succeeded past the traceIndex, or failed and - // potentially backtracked. + // potentially extended passed the [[traceIndex]], since that is the point + // at which all error reporting in Fastparse is focused. + // + // We determine that by only setting `shortParserMsg` if `newFailureGroups` + // is not empty. This works because: + // + // - Terminal parsers which report parse messages via `reportTerminalMsg` + // only have `setShortMsg=true` if `startIndex >= traceIndex` + // + // - Any aggregate parsers that build on top of them will thus only have + // `setShortMsg=true` if all of their sub-parsers' `startIndex >= traceIndex` + // + // - The same applies to aggregate parsers building on top of other + // aggregate parsers, and thus we can be confident that newFailureGroups.value.nonEmpty ) } diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index d12093c5..3387e577 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -31,15 +31,14 @@ object FailureTests extends TestSuite{ assert( trace.groupAggregateString == """(parseB | "c")""", - trace.terminalAggregateString == """("a" | "b" | "c" | "x")""" + trace.terminalAggregateString == """("a" | "b" | "c")""" ) } test("either") - check{ def parseB[$: P] = P( "a" | "b" ) def parseA[$: P] = P( (parseB | "") ~ "c" ) - def parse0[$: P] = P( ("" ~/ parseA).? ~ "x" ) - parse0(_) + parse0A(_) } test("option") - check{ def parseB[$: P] = P( "a" | "b" ) From d0cb9af78b00eeb6e36e68b30c029c88f8b12554 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:36:47 +0800 Subject: [PATCH 41/49] simplify --- .../src-2/fastparse/internal/MacroImpls.scala | 10 +++---- .../fastparse/internal/MacroRepImpls.scala | 2 +- .../src-2/fastparse/internal/RepImpls.scala | 8 ++--- .../fastparse/internal/MacroInlineImpls.scala | 10 +++---- .../fastparse/internal/MacroRepImpls.scala | 2 +- fastparse/src/fastparse/Parsed.scala | 4 +-- fastparse/src/fastparse/ParsingRun.scala | 29 +++++++++++------- .../src/fastparse/SharedPackageDefs.scala | 16 +++++----- fastparse/src/fastparse/internal/Util.scala | 2 +- .../test/src/fastparse/FailureTests.scala | 4 +-- readme/ErrorReportingInternals.scalatex | 30 +++++++++---------- 11 files changed, 62 insertions(+), 55 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 2a7a3ede..5c7bdba4 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -231,7 +231,7 @@ object MacroImpls { lhs0.splice val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureAggregates + val lhsAggregate = ctx5.aggregateParserMsgs if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -250,7 +250,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) + if (verboseFailures) ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -465,7 +465,7 @@ object MacroImpls { else { val $preRhsIndex = $ctx1.index $rhs - val $rhsAggregate = $ctx1.failureAggregates + val $rhsAggregate = $ctx1.aggregateParserMsgs val $rhsMsg = $ctx1.shortParserMsg val $res = if (!$ctx1.isSuccess) { @@ -491,7 +491,7 @@ object MacroImpls { $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, - // We override the failureAggregates to avoid building an `a ~ b` + // We override the aggregateParserMsgs to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -521,7 +521,7 @@ object MacroImpls { if (!$ctx1.isSuccess) $ctx1 else { val $postLhsIndex = $ctx1.index - val $lhsAggregate = $ctx1.failureAggregates + val $lhsAggregate = $ctx1.aggregateParserMsgs val $lhsMsg = $ctx1.shortParserMsg $setCut diff --git a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala index 9ccbc5bb..9978cca7 100644 --- a/fastparse/src-2/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroRepImpls.scala @@ -75,7 +75,7 @@ object MacroRepImpls{ ${c.prefix}.parse0() val $parsedMsg = $ctx1.shortParserMsg - val $parsedAgg = $ctx1.failureAggregates + val $parsedAgg = $ctx1.aggregateParserMsgs $originalCut |= $ctx1.cut if (!$ctx1.isSuccess) { val res = diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index 94872a08..3447dfa2 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -34,7 +34,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val verboseFailures = ctx.verboseFailures parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureAggregates + val parsedAgg = ctx.aggregateParserMsgs val postCut = ctx.cut if (!ctx.isSuccess) { val res = @@ -95,7 +95,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureAggregates + val parsedAgg = ctx.aggregateParserMsgs val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -156,7 +156,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ else { parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureAggregates + val parsedAgg = ctx.aggregateParserMsgs val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { @@ -224,7 +224,7 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ ctx.cut = precut | (count < min && outerCut) parse0() val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureAggregates + val parsedAgg = ctx.aggregateParserMsgs val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess){ diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index ee80c7dc..66932782 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -148,7 +148,7 @@ object MacroInlineImpls { if (!ctx1.isSuccess) ctx1 else { val postLhsIndex = ctx1.index - val lhsAggregate = ctx1.failureAggregates + val lhsAggregate = ctx1.aggregateParserMsgs val lhsMsg = ctx1.shortParserMsg ${ setCut('{ ctx1 }) } @@ -162,7 +162,7 @@ object MacroInlineImpls { else { val preRhsIndex = ctx1.index $rhs - val rhsAggregate = ctx1.failureAggregates + val rhsAggregate = ctx1.aggregateParserMsgs val rhsMsg = ctx1.shortParserMsg val res = if (!ctx1.isSuccess) { @@ -191,7 +191,7 @@ object MacroInlineImpls { preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, - // We override the failureAggregates to avoid building an `a ~ b` + // We override the aggregateParserMsgs to avoid building an `a ~ b` // aggregate msg in the specific case where the LHS parser fails to // make any progress past `startIndex`. This finds cases like `a.? ~ b` // or `a.rep ~ b` and lets use flatten them out into `a | b` @@ -298,7 +298,7 @@ object MacroInlineImpls { lhs0 val lhsMsg = ctx5.shortParserMsg - val lhsAggregate = ctx5.failureAggregates + val lhsAggregate = ctx5.aggregateParserMsgs if (ctx5.isSuccess) { ctx5.cut |= oldCut ctx5.asInstanceOf[ParsingRun[V]] @@ -317,7 +317,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) { - ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.failureAggregates ::: lhsAggregate) + ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) } ctx5.asInstanceOf[ParsingRun[V]] } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index a80d1598..90a0b774 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -98,7 +98,7 @@ object MacroRepImpls { else { $parse0 val parsedMsg = ctx.shortParserMsg - val parsedAgg = ctx.failureAggregates + val parsedAgg = ctx.aggregateParserMsgs val postCut = ctx.cut val verboseFailures = ctx.verboseFailures if (!ctx.isSuccess) { diff --git a/fastparse/src/fastparse/Parsed.scala b/fastparse/src/fastparse/Parsed.scala index a0993f61..0256387c 100644 --- a/fastparse/src/fastparse/Parsed.scala +++ b/fastparse/src/fastparse/Parsed.scala @@ -148,8 +148,8 @@ object Parsed{ def fromParsingRun[T](p: ParsingRun[T]) = { assert(!p.isSuccess) TracedFailure( - p.failureTerminals, - p.failureAggregates, + p.terminalParserMsgs, + p.aggregateParserMsgs, Parsed.fromParsingRun(p).asInstanceOf[Failure] ) } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index d10d7962..56cecdf3 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -35,10 +35,10 @@ import fastparse.internal.{Instrument, Lazy, Msgs, Util} * it with tracing enabled. * @param traceIndex The index we wish to trace if tracing is enabled, else * -1. Used to find failure messages to aggregate into - * `failureTerminals` + * `terminalParserMsgs` * @param instrument Callbacks that can be injected before/after every * `P(...)` parser. - * @param failureTerminals When tracing is enabled, this collects up all the + * @param terminalParserMsgs When tracing is enabled, this collects up all the * upper-most failures that happen at [[traceIndex]] * (in [[Lazy]] wrappers) so they can be shown to the * user at end-of-parse as suggestions for what could @@ -108,8 +108,8 @@ final class ParsingRun[+T](val input: ParserInput, val traceIndex: Int, val instrument: Instrument, // Mutable vars below: - var failureTerminals: Msgs, - var failureAggregates: Msgs, + var terminalParserMsgs: Msgs, + var aggregateParserMsgs: Msgs, var shortParserMsg: Msgs, var lastFailureMsg: Msgs, var failureStack: List[(String, Int)], @@ -143,7 +143,7 @@ final class ParsingRun[+T](val input: ParserInput, def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, failureAggregates) + reportAggregateMsg(startIndex, newShortParserMsg, aggregateParserMsgs) } def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, @@ -155,7 +155,7 @@ final class ParsingRun[+T](val input: ParserInput, def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, failureAggregates, forceAggregate) + reportAggregateMsg(startIndex, newShortParserMsg, aggregateParserMsgs, forceAggregate) } def reportAggregateMsg(startIndex: Int, @@ -182,7 +182,14 @@ final class ParsingRun[+T](val input: ParserInput, // `setShortMsg=true` if all of their sub-parsers' `startIndex >= traceIndex` // // - The same applies to aggregate parsers building on top of other - // aggregate parsers, and thus we can be confident that + // aggregate parsers + // + // - Thus we can be confident that all our parsers only have `setShortMsg=true + // if their `startIndex >= traceIndex` + // + // - The only exception to this is some parsers which use `forceAggregate=true` + // when only some of their sub-parsers' `startIndex >= traceIndex`, such + // that those sub-parsers can be propagated up as `aggregateParserMsgs` newFailureGroups.value.nonEmpty ) } @@ -197,14 +204,14 @@ final class ParsingRun[+T](val input: ParserInput, * - Parsers like `.opaque`, where sub-failures are intentionally hidden and * not shown to the user * - * These "terminal" failures will be stored in the `failureTerminals` in case + * These "terminal" failures will be stored in the `terminalParserMsgs` in case * a user wants to know what could have been placed at the failure point to * let the parse progress */ def reportTerminalMsg(startIndex: Int, newShortParserMsg: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex - if (!isSuccess && index == traceIndex) failureTerminals :::= newShortParserMsg + if (!isSuccess && index == traceIndex) terminalParserMsgs :::= newShortParserMsg reportParseMsg0( startIndex, @@ -225,9 +232,9 @@ final class ParsingRun[+T](val input: ParserInput, shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty // There are two cases when aggregating: either we stomp over the entire - // existing `failureAggregates` with `newShortParserMsg`, or we preserve it + // existing `aggregateParserMsgs` with `newShortParserMsg`, or we preserve it // (with possible additions) with `newFailureGroups`. - failureAggregates = + aggregateParserMsgs = if (forceAggregate) newFailureGroups else if (discardNewFailureGroups(startIndex)) shortParserMsg else newFailureGroups diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 79801087..9f4fddec 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -54,8 +54,8 @@ trait SharedPackageDefs { originalParser = parser, traceIndex = traceIndex, instrument = instrument, - failureTerminals = Msgs.empty, - failureAggregates = Msgs.empty, + terminalParserMsgs = Msgs.empty, + aggregateParserMsgs = Msgs.empty, shortParserMsg = Msgs.empty, lastFailureMsg = null, failureStack = List.empty, @@ -105,7 +105,7 @@ trait SharedPackageDefs { if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) else ctx.asInstanceOf[P[Unit]] if (ctx.verboseFailures) { - ctx.failureAggregates = Msgs.empty + ctx.aggregateParserMsgs = Msgs.empty ctx.reportTerminalMsg(startPos, if (msg.value.isEmpty) Msgs.empty else () => msg match{ @@ -145,7 +145,7 @@ trait SharedPackageDefs { /** * Wraps a parser and ensures that none of the parsers within it leave - * failure traces in failureTerminals, though unlike [[ByNameOps.opaque]] + * failure traces in terminalParserMsgs, though unlike [[ByNameOps.opaque]] * if there is a failure *within* the wrapped parser the failure's location * and error message will still be shown * @@ -157,7 +157,7 @@ trait SharedPackageDefs { val res = p if (ctx.verboseFailures) { - ctx.failureAggregates = Msgs.empty + ctx.aggregateParserMsgs = Msgs.empty ctx.shortParserMsg = Msgs.empty } res @@ -273,7 +273,7 @@ object SharedPackageDefs{ val startCut = ctx.cut val oldNoCut = ctx.noDropBuffer ctx.noDropBuffer = true - val startTerminals = ctx.failureTerminals + val startTerminals = ctx.terminalParserMsgs parse0() ctx.noDropBuffer = oldNoCut val msg = ctx.shortParserMsg @@ -283,8 +283,8 @@ object SharedPackageDefs{ else ctx.freshSuccessUnit(startPos) if (ctx.verboseFailures) { - ctx.failureTerminals = startTerminals - ctx.failureAggregates = Msgs.empty + ctx.terminalParserMsgs = startTerminals + ctx.aggregateParserMsgs = Msgs.empty ctx.reportTerminalMsg(startPos, Msgs.empty) } res.cut = startCut diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index d2b94128..1e312535 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -122,7 +122,7 @@ object Util { // we backtrack past the sep on failure) as well as the failure // aggregate of the previous rep, which we could have continued val newAgg = - if (sepMsg == null || precut) ctx.failureAggregates + if (sepMsg == null || precut) ctx.aggregateParserMsgs else Util.joinBinOp(sepMsg, parsedMsg) ctx.reportAggregateMsg( diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 3387e577..ad379e8c 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -38,7 +38,7 @@ object FailureTests extends TestSuite{ test("either") - check{ def parseB[$: P] = P( "a" | "b" ) def parseA[$: P] = P( (parseB | "") ~ "c" ) - parse0A(_) + parseA(_) } test("option") - check{ def parseB[$: P] = P( "a" | "b" ) @@ -434,7 +434,7 @@ object FailureTests extends TestSuite{ import NoWhitespace._ // In the case where one branch fails further in than `traceIndex`, we // collect the partial aggregation from that branch in the - // `failureAggregates` but ignore that branch's downstream failure in + // `aggregateParserMsgs` but ignore that branch's downstream failure in // `failureTerminalsAggregate` def check(parser: P[_] => P[_]) = checkOffset( diff --git a/readme/ErrorReportingInternals.scalatex b/readme/ErrorReportingInternals.scalatex index 1552b0ad..2bbd35e9 100644 --- a/readme/ErrorReportingInternals.scalatex +++ b/readme/ErrorReportingInternals.scalatex @@ -20,28 +20,28 @@ levels will treat this as follows: @ul @li - @code{failureTerminals} lists all the lowest-level terminal parsers which are + @code{terminalParserMsgs} lists all the lowest-level terminal parsers which are tried at the given @code{traceIndex}, i.e. the character class @code{[0-9]} and the token @hl.scala{"("}. This is useful to answer the question "what token/char can I put at the error position to make my parse continue". The implementation - of @code{failureTerminals} is straightforward: we simply call + of @code{terminalParserMsgs} is straightforward: we simply call @code{reportTerminalMsg} in every terminal parser, which collects all the messages in a big list and returns it. @li - @code{failureAggregates} lists all high-level parsers which are tried at the given + @code{aggregateParserMsgs} lists all high-level parsers which are tried at the given @code{traceIndex}, i.e. the named parsers @code{num} and @code{plus}. This is useful to answer the question "What construct was the parser trying to do when it failed" @p - The implementation of @code{failureAggregates} is more interesting, since we need + The implementation of @code{aggregateParserMsgs} is more interesting, since we need to define what "high level" parsers means, which is non-obvious. - @sect{Definition of failureAggregates} + @sect{Definition of aggregateParserMsgs} @p - Fastparse uses the following definition for @code{failureAggregates}: + Fastparse uses the following definition for @code{aggregateParserMsgs}: @ul @li - @code{failureAggregates} should contain the parsers highest in the call stack, + @code{aggregateParserMsgs} should contain the parsers highest in the call stack, whose failure isn't immediately fatal to the parse (due to them being in @code{|}, @code{.rep}, @code{?}, or other "backtrackable" operators, but not past a @code{cut}) @@ -49,7 +49,7 @@ This is a useful definition because we already have the @code{failureStack} containing all (named) parsers whose failure *is* immediately fatal to the parse, both those at @code{traceIndex} and those earlier in the input. Thus - there is no need to duplicate showing any of them in the @code{failureAggregates}, + there is no need to duplicate showing any of them in the @code{aggregateParserMsgs}, and we can instead go "one level deeper" to find the highest-level parsers within the deepest parser of the @code{failureStack} and show those instead. Thus, in the combined @code{longAggregateMsg}, the failure stack shows us @@ -57,32 +57,32 @@ @code{traceIndex}, while the longAggregateMsg tells us what are the highest-level parsers FastParse was trying to parse at @code{traceIndex} before it finally failed. - @sect{Implementation of failureAggregates} + @sect{Implementation of aggregateParserMsgs} @p - To collect the @code{failureAggregates}, We use the following algorithm: + To collect the @code{aggregateParserMsgs}, We use the following algorithm: @ul @li When a parse which started at the given @code{traceIndex} fails without a cut: - Over-write @code{failureAggregates} with it's @code{shortParserMsg} + Over-write @code{aggregateParserMsgs} with it's @code{shortParserMsg} @li Otherwise: @ul @li - If we are a terminal parser, we set our @code{failureAggregates} to Nil + If we are a terminal parser, we set our @code{aggregateParserMsgs} to Nil @li - If we are a compound parser, we simply sum up the @code{failureAggregates} + If we are a compound parser, we simply sum up the @code{aggregateParserMsgs} of all our constituent parts @p As mentioned earlier, the point of this is to provide the highest-level parsers which failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. non-highest-level parsers do successfully write their message to - @code{failureAggregates}, but they are subsequently over-written by the higher + @code{aggregateParserMsgs}, but they are subsequently over-written by the higher level parsers, until it reaches the point where @code{cut == true}, indicating that any further higher-level parsers will be in @code{failureStack} and using their message to stomp over the existing parse-failure-messages in - @code{failureAggregates} would be wasteful. + @code{aggregateParserMsgs} would be wasteful. @sect{Edge Cases} @p These is an edge case where there is no given failure that occurs exactly at From 856711e799d64891af3b1b5d0327738236ff0e84 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:39:53 +0800 Subject: [PATCH 42/49] simplify --- fastparse/src/fastparse/ParsingRun.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 56cecdf3..093ac248 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -236,14 +236,14 @@ final class ParsingRun[+T](val input: ParserInput, // (with possible additions) with `newFailureGroups`. aggregateParserMsgs = if (forceAggregate) newFailureGroups - else if (discardNewFailureGroups(startIndex)) shortParserMsg + else if (shouldAggregate(startIndex)) shortParserMsg else newFailureGroups } /** * Conditions under which we want to aggregate the given parse */ - def discardNewFailureGroups(startIndex: Int) = { + def shouldAggregate(startIndex: Int) = { // We only aggregate if we are not currently past a cut; if we are past a // cut, there is no further backtracking and so the error aggregate that has // occurred will be the final aggregate shown to the user From b7e9abcba5d7b27db183e751c515692ab559ea5f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:45:27 +0800 Subject: [PATCH 43/49] cleanup --- fastparse/src/fastparse/ParsingRun.scala | 3 +++ readme/ErrorReportingInternals.scalatex | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index 093ac248..cfb4d77a 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -227,6 +227,9 @@ final class ParsingRun[+T](val input: ParserInput, newFailureGroups: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { + // `lastFailureMsg` ends up being set by the first parser to report a + // failure, while returning from the last parser to call `.freshFailure() + // (which nulls it out) if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty diff --git a/readme/ErrorReportingInternals.scalatex b/readme/ErrorReportingInternals.scalatex index 2bbd35e9..ecb9154c 100644 --- a/readme/ErrorReportingInternals.scalatex +++ b/readme/ErrorReportingInternals.scalatex @@ -104,7 +104,7 @@ follows the failure). This then gets aggregated nicely to form the error message from-point-of-failure. @p - A follow-on edge case is parsing "ax" with + A follow-on edge case is parsing @hl.scala{"ax"} with @hl.scala val inner = P( "a" ~ "b" ) P( inner ~ "c" | "a" ~/ "d" ) From c61207821787a441de9d2d86122c07f61f071971 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 5 Mar 2023 17:50:13 +0800 Subject: [PATCH 44/49] simplify --- fastparse/src/fastparse/ParsingRun.scala | 51 ++++++++++-------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index cfb4d77a..acb7867d 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -147,9 +147,9 @@ final class ParsingRun[+T](val input: ParserInput, } def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, - newFailureGroups: Msgs): Unit = { + newAggregateMsgs: Msgs): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, newFailureGroups, false) + reportAggregateMsg(startIndex, newShortParserMsg, newAggregateMsgs, false) } def reportAggregateMsg(startIndex: Int, @@ -160,37 +160,15 @@ final class ParsingRun[+T](val input: ParserInput, def reportAggregateMsg(startIndex: Int, newShortParserMsg: Msgs, - newFailureGroups: Msgs, + newAggregateMsgs: Msgs, forceAggregate: Boolean): Unit = { reportParseMsg0( startIndex, newShortParserMsg, - newFailureGroups, + newAggregateMsgs, forceAggregate, - // We only want to set the shortMsg for most parsers if they could have - // potentially extended passed the [[traceIndex]], since that is the point - // at which all error reporting in Fastparse is focused. - // - // We determine that by only setting `shortParserMsg` if `newFailureGroups` - // is not empty. This works because: - // - // - Terminal parsers which report parse messages via `reportTerminalMsg` - // only have `setShortMsg=true` if `startIndex >= traceIndex` - // - // - Any aggregate parsers that build on top of them will thus only have - // `setShortMsg=true` if all of their sub-parsers' `startIndex >= traceIndex` - // - // - The same applies to aggregate parsers building on top of other - // aggregate parsers - // - // - Thus we can be confident that all our parsers only have `setShortMsg=true - // if their `startIndex >= traceIndex` - // - // - The only exception to this is some parsers which use `forceAggregate=true` - // when only some of their sub-parsers' `startIndex >= traceIndex`, such - // that those sub-parsers can be propagated up as `aggregateParserMsgs` - newFailureGroups.value.nonEmpty + newAggregateMsgs.value.nonEmpty ) } @@ -224,7 +202,7 @@ final class ParsingRun[+T](val input: ParserInput, def reportParseMsg0(startIndex: Int, newShortParserMsg: Msgs, - newFailureGroups: Msgs, + newAggregateMsgs: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { // `lastFailureMsg` ends up being set by the first parser to report a @@ -232,15 +210,26 @@ final class ParsingRun[+T](val input: ParserInput, // (which nulls it out) if (!isSuccess && lastFailureMsg == null) lastFailureMsg = newShortParserMsg + // We only set the `shortParserMsg` for some parsers. These include: + // + // - Terminal parsers which have `startIndex >= traceIndex` + // + // - Aggregate parsers which have non-empty `newAggregateMsgs`, indicating + // that they have either child terminal parsers with `startIndex >= traceIndex` + // or they have child aggregate parsers with non-empty `newAggregateMsgs` + // + // This lets us skip setting `shortParserMsg` for all parsers, terminal or + // aggregate, which run and terminate fully before `traceIndex`, and thus + // would be of no interest to a user debugging parse failures at `traceIndex` shortParserMsg = if (setShortMsg) newShortParserMsg else Msgs.empty // There are two cases when aggregating: either we stomp over the entire // existing `aggregateParserMsgs` with `newShortParserMsg`, or we preserve it - // (with possible additions) with `newFailureGroups`. + // (with possible additions) with `newAggregateMsgs`. aggregateParserMsgs = - if (forceAggregate) newFailureGroups + if (forceAggregate) newAggregateMsgs else if (shouldAggregate(startIndex)) shortParserMsg - else newFailureGroups + else newAggregateMsgs } /** From f3dc7536a57a360acec7f97e61ed67ec7862a802 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 7 Mar 2023 14:00:38 +0800 Subject: [PATCH 45/49] . --- .../src-2/fastparse/internal/MacroImpls.scala | 8 +-- .../src-2/fastparse/internal/RepImpls.scala | 6 +- .../fastparse/internal/MacroInlineImpls.scala | 8 +-- .../fastparse/internal/MacroRepImpls.scala | 3 +- fastparse/src/fastparse/ParsingRun.scala | 60 ++++++---------- fastparse/src/fastparse/Whitespace.scala | 71 ++++++++++++++----- fastparse/src/fastparse/internal/Util.scala | 1 - .../src/scalaparse/unit/FailureTests.scala | 6 +- 8 files changed, 91 insertions(+), 72 deletions(-) diff --git a/fastparse/src-2/fastparse/internal/MacroImpls.scala b/fastparse/src-2/fastparse/internal/MacroImpls.scala index 5c7bdba4..953e8753 100644 --- a/fastparse/src-2/fastparse/internal/MacroImpls.scala +++ b/fastparse/src-2/fastparse/internal/MacroImpls.scala @@ -53,7 +53,6 @@ object MacroImpls { } if (ctx0.verboseFailures) { ctx0.reportAggregateMsg( - startIndex, () => name.splice.value, forceAggregate = startIndex < ctx0.traceIndex ) @@ -241,7 +240,7 @@ object MacroImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportAggregateMsg(startPos, lhsMsg) + if (verboseFailures) ctx5.reportAggregateMsg(lhsMsg) ctx5.cut = false other.splice @@ -250,7 +249,7 @@ object MacroImpls { val endCut = rhsCut | oldCut if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut - if (verboseFailures) ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) + if (verboseFailures) ctx5.reportAggregateMsg(rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) ctx5.asInstanceOf[ParsingRun[V]] } } @@ -488,7 +487,6 @@ object MacroImpls { } if ($ctx1.verboseFailures) $ctx1.reportAggregateMsg( - $preLhsIndex, _root_.fastparse.internal.Util.joinBinOp($lhsMsg, $rhsMsg), $rhsAggregate ::: $lhsAggregate, // We override the aggregateParserMsgs to avoid building an `a ~ b` @@ -693,7 +691,7 @@ object MacroImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg if (!postSuccess){ - ctx1.reportAggregateMsg(startPos, () => msg.render + ".?") + ctx1.reportAggregateMsg(() => msg.render + ".?") } } res diff --git a/fastparse/src-2/fastparse/internal/RepImpls.scala b/fastparse/src-2/fastparse/internal/RepImpls.scala index 3447dfa2..403ae0d8 100644 --- a/fastparse/src-2/fastparse/internal/RepImpls.scala +++ b/fastparse/src-2/fastparse/internal/RepImpls.scala @@ -182,9 +182,10 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val endCut = outerCut | postCut | sepCut if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) else if (ctx.isSuccess) { + val sepMsg = ctx.shortParserMsg if (!consumeWhitespace(whitespace, ctx, sepCut)) ctx.asInstanceOf[ParsingRun[Nothing]] else { - rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + rec(beforeSepIndex, nextCount, sepCut, endCut, sepMsg, parsedAgg) } } else { @@ -245,9 +246,10 @@ class RepImpls[T](val parse0: () => ParsingRun[T]) extends AnyVal{ val endCut = outerCut | postCut | sepCut if (sep1 == null) rec(beforeSepIndex, nextCount, false, endCut, null, parsedAgg) else if (ctx.isSuccess) { + val sepMsg = ctx.shortParserMsg if (!consumeWhitespace(whitespace, ctx, sepCut)) ctx.asInstanceOf[ParsingRun[Nothing]] else { - rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + rec(beforeSepIndex, nextCount, sepCut, endCut, sepMsg, parsedAgg) } } else { diff --git a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala index 66932782..9df8c1df 100644 --- a/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroInlineImpls.scala @@ -104,7 +104,6 @@ object MacroInlineImpls { } if (ctx0.verboseFailures) { ctx0.reportAggregateMsg( - startIndex, () => name.value, forceAggregate = startIndex < ctx0.traceIndex ) @@ -188,7 +187,6 @@ object MacroInlineImpls { } if (ctx1.verboseFailures) ctx1.reportAggregateMsg( - preLhsIndex, Util.joinBinOp(lhsMsg, rhsMsg), rhsAggregate ::: lhsAggregate, // We override the aggregateParserMsgs to avoid building an `a ~ b` @@ -244,7 +242,7 @@ object MacroInlineImpls { if (ctx1.verboseFailures) { val msg = ctx1.shortParserMsg if (!postSuccess) { - ctx1.reportAggregateMsg(startPos, () => msg.render + ".?") + ctx1.reportAggregateMsg(() => msg.render + ".?") } } res @@ -307,7 +305,7 @@ object MacroInlineImpls { val verboseFailures = ctx5.verboseFailures ctx5.index = startPos - if (verboseFailures) ctx5.reportAggregateMsg(startPos, lhsMsg) + if (verboseFailures) ctx5.reportAggregateMsg(lhsMsg) ctx5.cut = false other @@ -317,7 +315,7 @@ object MacroInlineImpls { if (!ctx5.isSuccess && !rhsCut) ctx5.freshFailure(startPos) ctx5.cut = endCut if (verboseFailures) { - ctx5.reportAggregateMsg(startPos, rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) + ctx5.reportAggregateMsg(rhsMsg ::: lhsMsg, ctx5.aggregateParserMsgs ::: lhsAggregate) } ctx5.asInstanceOf[ParsingRun[V]] } diff --git a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala index 90a0b774..30a1367d 100644 --- a/fastparse/src-3/fastparse/internal/MacroRepImpls.scala +++ b/fastparse/src-3/fastparse/internal/MacroRepImpls.scala @@ -148,9 +148,10 @@ object MacroRepImpls { val sepCut = ctx.cut val endCut = outerCut | postCut | sepCut if (ctx.isSuccess) { + val postSepMsg = ctx.shortParserMsg ${ consumeWhitespace('{sepCut})('{ - rec(beforeSepIndex, nextCount, sepCut, endCut, ctx.shortParserMsg, parsedAgg) + rec(beforeSepIndex, nextCount, sepCut, endCut, postSepMsg, parsedAgg) }) } } diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index acb7867d..bcee5fcb 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -140,31 +140,26 @@ final class ParsingRun[+T](val input: ParserInput, * of inlining in Fastparse, and large amounts of bytecode inlined in a method * can cause JVM performance problems (e.g. JIT compilation may get disabled) */ - def reportAggregateMsg(startIndex: Int, - newShortParserMsg: Msgs): Unit = { + def reportAggregateMsg(newShortParserMsg: Msgs): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, aggregateParserMsgs) + reportAggregateMsg(newShortParserMsg, aggregateParserMsgs) } - def reportAggregateMsg(startIndex: Int, - newShortParserMsg: Msgs, + def reportAggregateMsg(newShortParserMsg: Msgs, newAggregateMsgs: Msgs): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, newAggregateMsgs, false) + reportAggregateMsg(newShortParserMsg, newAggregateMsgs, false) } - def reportAggregateMsg(startIndex: Int, - newShortParserMsg: Msgs, + def reportAggregateMsg(newShortParserMsg: Msgs, forceAggregate: Boolean): Unit = { - reportAggregateMsg(startIndex, newShortParserMsg, aggregateParserMsgs, forceAggregate) + reportAggregateMsg(newShortParserMsg, aggregateParserMsgs, forceAggregate) } - def reportAggregateMsg(startIndex: Int, - newShortParserMsg: Msgs, + def reportAggregateMsg(newShortParserMsg: Msgs, newAggregateMsgs: Msgs, forceAggregate: Boolean): Unit = { reportParseMsg0( - startIndex, newShortParserMsg, newAggregateMsgs, forceAggregate, @@ -186,13 +181,11 @@ final class ParsingRun[+T](val input: ParserInput, * a user wants to know what could have been placed at the failure point to * let the parse progress */ - def reportTerminalMsg(startIndex: Int, - newShortParserMsg: Msgs): Unit = { + def reportTerminalMsg(newShortParserMsg: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex if (!isSuccess && index == traceIndex) terminalParserMsgs :::= newShortParserMsg reportParseMsg0( - startIndex, if (startIndex >= traceIndex) newShortParserMsg else Msgs.empty, if (startIndex >= traceIndex) newShortParserMsg else Msgs.empty, false, @@ -200,8 +193,7 @@ final class ParsingRun[+T](val input: ParserInput, ) } - def reportParseMsg0(startIndex: Int, - newShortParserMsg: Msgs, + def reportParseMsg0(newShortParserMsg: Msgs, newAggregateMsgs: Msgs, forceAggregate: Boolean, setShortMsg: Boolean): Unit = { @@ -228,29 +220,23 @@ final class ParsingRun[+T](val input: ParserInput, // (with possible additions) with `newAggregateMsgs`. aggregateParserMsgs = if (forceAggregate) newAggregateMsgs - else if (shouldAggregate(startIndex)) shortParserMsg + // We only replace the aggregate Msgs if: + // + // 1. We are not currently past a cut; if we are past a cut, there is no + // further backtracking and so the error aggregate that has occurred + // will be the final aggregate shown to the user + // + // 2. Only replace in case of failures + // + // 3. Only stomp over the given aggregation with shortParserMsg if the + // current parser has failed and the final parse `index` (after any + // backtracking) is still at-or-greater-than the `traceIndex`. That + // ensures that any parsers which started/ended before the point of + // failure are not shown, since they are irrelevant + else if (!cut && !isSuccess && traceIndex <= index) shortParserMsg else newAggregateMsgs } - /** - * Conditions under which we want to aggregate the given parse - */ - def shouldAggregate(startIndex: Int) = { - // We only aggregate if we are not currently past a cut; if we are past a - // cut, there is no further backtracking and so the error aggregate that has - // occurred will be the final aggregate shown to the user - !cut && - // Only aggregate failures - !isSuccess && - // We only stomp over the given aggregation with shortParserMsg if the range - // of the failed parse surrounds `traceIndex`. For parses that occur - // completely before or after the `traceIndex`, the actual parse doesn't - // contribute anything to the aggregation. - startIndex <= traceIndex && - traceIndex <= index - } - - // Use telescoping methods rather than default arguments to try and minimize // the amount of bytecode generated at the callsite. // diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index f07bc3cb..e97ae013 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -4,7 +4,7 @@ import fastparse._ import fastparse.internal.Util import scala.annotation.{Annotation, switch, tailrec} - +import fastparse.internal.Msgs trait Whitespace{ def apply(ctx: ParsingRun[_]): ParsingRun[Unit] @@ -32,6 +32,7 @@ object SingleLineWhitespace { input.isReachable(index) && (input(index) match{ case ' ' | '\t' => true case _ => false}) ) index += 1 + if (ctx.verboseFailures) ctx.reportTerminalMsg(index, Msgs.empty) ctx.freshSuccessUnit(index = index) } } @@ -50,6 +51,7 @@ object MultiLineWhitespace { input.isReachable(index) && (input(index) match{ case ' ' | '\t' | '\r' | '\n' => true case _ => false}) ) index += 1 + if (ctx.verboseFailures) ctx.reportTerminalMsg(index, Msgs.empty) ctx.freshSuccessUnit(index = index) } } @@ -64,7 +66,10 @@ object ScriptWhitespace{ def apply(ctx: ParsingRun[_]) = { val input = ctx.input @tailrec def rec(current: Int, state: Int): ParsingRun[Unit] = { - if (!input.isReachable(current)) ctx.freshSuccessUnit(current) + if (!input.isReachable(current)) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) + } else { val currentChar = input(current) (state: @switch) match{ @@ -72,7 +77,9 @@ object ScriptWhitespace{ (currentChar: @switch) match{ case ' ' | '\t' | '\n' | '\r' => rec(current + 1, state) case '#' => rec(current + 1, state = 1) - case _ => ctx.freshSuccessUnit(current) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) } case 1 => rec(current + 1, state = if (currentChar == '\n') 0 else state) } @@ -92,11 +99,16 @@ object JavaWhitespace{ implicit object whitespace extends Whitespace { def apply(ctx: ParsingRun[_]) = { val input = ctx.input - val startIndex = ctx.index @tailrec def rec(current: Int, state: Int): ParsingRun[Unit] = { if (!input.isReachable(current)) { - if (state == 0 || state == 1) ctx.freshSuccessUnit(current) - else if(state == 2) ctx.freshSuccessUnit(current - 1) + if (state == 0 || state == 1) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) + } + else if(state == 2) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) + } else { ctx.cut = true val res = ctx.freshFailure(current) @@ -110,14 +122,18 @@ object JavaWhitespace{ (currentChar: @switch) match{ case ' ' | '\t' | '\n' | '\r' => rec(current + 1, state) case '/' => rec(current + 1, state = 2) - case _ => ctx.freshSuccessUnit(current) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) } case 1 => rec(current + 1, state = if (currentChar == '\n') 0 else state) case 2 => (currentChar: @switch) match{ case '/' => rec(current + 1, state = 1) case '*' => rec(current + 1, state = 3) - case _ => ctx.freshSuccessUnit(current - 1) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) } case 3 => rec(current + 1, state = if (currentChar == '*') 4 else state) case 4 => @@ -144,11 +160,16 @@ object JsonnetWhitespace{ implicit object whitespace extends Whitespace { def apply(ctx: ParsingRun[_]) = { val input = ctx.input - val startIndex = ctx.index @tailrec def rec(current: Int, state: Int): ParsingRun[Unit] = { if (!input.isReachable(current)) { - if (state == 0 || state == 1) ctx.freshSuccessUnit(current) - else if(state == 2) ctx.freshSuccessUnit(current - 1) + if (state == 0 || state == 1) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) + } + else if(state == 2) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) + } else { ctx.cut = true val res = ctx.freshFailure(current) @@ -164,14 +185,18 @@ object JsonnetWhitespace{ case ' ' | '\t' | '\n' | '\r' => rec(current + 1, state) case '#' => rec(current + 1, state = 1) case '/' => rec(current + 1, state = 2) - case _ => ctx.freshSuccessUnit(current) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) } case 1 => rec(current + 1, state = if (currentChar == '\n') 0 else state) case 2 => (currentChar: @switch) match{ case '/' => rec(current + 1, state = 1) case '*' => rec(current + 1, state = 3) - case _ => ctx.freshSuccessUnit(current - 1) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) } case 3 => rec(current + 1, state = if (currentChar == '*') 4 else state) case 4 => @@ -197,11 +222,16 @@ object ScalaWhitespace { implicit object whitespace extends Whitespace { def apply(ctx: ParsingRun[_]) = { val input = ctx.input - val startIndex = ctx.index @tailrec def rec(current: Int, state: Int, nesting: Int): ParsingRun[Unit] = { if (!input.isReachable(current)) { - if (state == 0 || state == 1) ctx.freshSuccessUnit(current) - else if(state == 2 && nesting == 0) ctx.freshSuccessUnit(current - 1) + if (state == 0 || state == 1) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) + } + else if(state == 2 && nesting == 0) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) + } else { ctx.cut = true val res = ctx.freshFailure(current) @@ -215,7 +245,9 @@ object ScalaWhitespace { (currentChar: @switch) match{ case ' ' | '\t' | '\n' | '\r' => rec(current + 1, state, 0) case '/' => rec(current + 1, state = 2, 0) - case _ => ctx.freshSuccessUnit(current) + case _ => + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) } case 1 => rec(current + 1, state = if (currentChar == '\n') 0 else state, 0) case 2 => @@ -225,7 +257,10 @@ object ScalaWhitespace { else rec(current + 1, state = 2, nesting) case '*' => rec(current + 1, state = 3, nesting + 1) case _ => - if (nesting == 0) ctx.freshSuccessUnit(current - 1) + if (nesting == 0) { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current - 1) + } else rec(current + 1, state = 3, nesting) } case 3 => diff --git a/fastparse/src/fastparse/internal/Util.scala b/fastparse/src/fastparse/internal/Util.scala index 1e312535..c26b9f11 100644 --- a/fastparse/src/fastparse/internal/Util.scala +++ b/fastparse/src/fastparse/internal/Util.scala @@ -126,7 +126,6 @@ object Util { else Util.joinBinOp(sepMsg, parsedMsg) ctx.reportAggregateMsg( - startIndex, () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), if (lastAgg == null) newAgg else newAgg ::: lastAgg diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index 66ee2784..dfa1981d 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -220,7 +220,7 @@ object FailureTests extends TestSuite{ |} | """.stripMargin, - aggregate = """("." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "}")""", + aggregate = """("." | TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found = ")" ) @@ -807,7 +807,7 @@ object FailureTests extends TestSuite{ s""" |object X{(2,)} """.stripMargin, - aggregate = """(FloatSuffix | "L" | "l" | WL ~ "." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "," ~ Expr | "," | ")")""", + aggregate = """(FloatSuffix | "L" | "l" | "." | TypeArgs | ArgList | `_` | InfixSuffix | PostFix | "=>" | `=` | MatchAscriptionSuffix | "," ~ Expr | "," | ")")""", terminals = null, found = ",)" ) @@ -943,7 +943,7 @@ object FailureTests extends TestSuite{ | for(i <- Nil if x: Int => bar) 1 |} """.stripMargin, - aggregate = """(TQ | "\"" | "." | WL ~ "." | WL ~ TypeArgs | ArgList | `_` | InfixSuffix | PostFix | Enumerator | ")")""", + aggregate = """(TQ | "\"" | "." | TypeArgs | ArgList | `_` | InfixSuffix | PostFix | Enumerator | ")")""", terminals = null, found = ": Int" ) From ecebc6dee631589a8f665cbdf4d2f8ddcfbb38a0 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 7 Mar 2023 14:01:50 +0800 Subject: [PATCH 46/49] . --- fastparse/src/fastparse/ParsingRun.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastparse/src/fastparse/ParsingRun.scala b/fastparse/src/fastparse/ParsingRun.scala index bcee5fcb..529aea7e 100644 --- a/fastparse/src/fastparse/ParsingRun.scala +++ b/fastparse/src/fastparse/ParsingRun.scala @@ -181,7 +181,7 @@ final class ParsingRun[+T](val input: ParserInput, * a user wants to know what could have been placed at the failure point to * let the parse progress */ - def reportTerminalMsg(newShortParserMsg: Msgs): Unit = { + def reportTerminalMsg(startIndex: Int, newShortParserMsg: Msgs): Unit = { // We only care about terminal parsers which failed exactly at the traceIndex if (!isSuccess && index == traceIndex) terminalParserMsgs :::= newShortParserMsg From e9882f9343b73680ca192726b191dbde7f42eb4a Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 7 Mar 2023 14:05:37 +0800 Subject: [PATCH 47/49] positive lookahead fixes --- .../src/fastparse/SharedPackageDefs.scala | 11 +--- .../test/src/fastparse/FailureTests.scala | 65 +++++++++++++++++-- .../src/scalaparse/unit/FailureTests.scala | 8 +-- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 9f4fddec..616a8cba 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -104,16 +104,7 @@ trait SharedPackageDefs { val res = if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) else ctx.asInstanceOf[P[Unit]] - if (ctx.verboseFailures) { - ctx.aggregateParserMsgs = Msgs.empty - ctx.reportTerminalMsg(startPos, - if (msg.value.isEmpty) Msgs.empty - else () => msg match{ - case Seq(x) => s"&(${msg.render})" - case xs => s"&${msg.render}" - } - ) - } + res.cut = startCut res } diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index ad379e8c..1dca56a9 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -272,13 +272,70 @@ object FailureTests extends TestSuite{ label = "end-of-input", terminals = """("y" | end-of-input)""", parser = { - def c[$: P] = P( "x".repX(1, "y") ).log - def d[$: P] = P( "p" ).log - def b[$: P] = P( (d ~ "t").repX(1, " ") ).log - def a[$: P] = P( b ~ " " ~ c ~ End ).log + def c[$: P] = P( "x".repX(1, "y") ) + def d[$: P] = P( "p" ) + def b[$: P] = P( (d ~ "t").repX(1, " ") ) + def a[$: P] = P( b ~ " " ~ c ~ End ) a(_) } ) + + test("lookahead") { + // We do not bother showing the enclosing `&()` for positive lookahead + // parsers. That is because to a user debugging the parser, it doesn't + // matter: whether the parser is `&(foo)` or `foo`, they still need to + // put the same input at `traceIndex` to make the parse succeed + // + // Furthermore, for both positive and negative lookahead which are + // typically used in a `&(lhs) ~ rhs` or `!lhs ~ rhs`, we cannot show + // the `rhs` even if we wanted to! The parse will already have failed + // when parsing the `lhs`, and so there is no opportunity to gather + // the `rhs`'s parse messages for display. + test("positive") - checkOffset( + input = "7", + expected = """[0-6]""", + label = "[0-6]", + terminals = """[0-6]""", + parser = { + def parse[$: P] = P( &(CharIn("0-6")) ~ CharIn("4-9") ~ End ) + parse(_) + } + ) +// test("negative") - checkOffset( +// input = "5", +// expected = """![0-6]""", +// label = "![0-6]", +// terminals = """![0-6]""", +// parser = { +// def parse[$: P] = P( !CharIn("0-6") ~ CharIn("4-9") ~ End) +// parse(_) +// } +// ) +// test("negative2") - checkOffset( +// input = "5", +// expected = """!([0-4] | [5-9])""", +// label = "!([0-4] | [5-9])", +// terminals = """!([0-4] | [5-9])""", +// parser = { +// // Make sure that the failure if `[0-4]` inside the `!(...)` block +// // does not end up in our reported terminals. The parser *wants* +// // the wrapped parser to fail, and giving hints to make its +// // sub-parsers succeed is counter-productive! +// def parse[$: P] = P( !(CharIn("0-4") | CharIn("5-9")) ~ End) +// parse(_) +// } +// ) + test("negative3") - checkOffset( + input = "9", + expected = """[4-8]""", + label = "[4-8]", + terminals = """[4-8]""", + parser = { + def parse[$: P] = P( !CharIn("0-6").log("lhs") ~ CharIn("4-8").log("rhs") ~ End ).log + parse(_) + } + ) + } } test("offset"){ diff --git a/scalaparse/test/src/scalaparse/unit/FailureTests.scala b/scalaparse/test/src/scalaparse/unit/FailureTests.scala index dfa1981d..3249dd65 100644 --- a/scalaparse/test/src/scalaparse/unit/FailureTests.scala +++ b/scalaparse/test/src/scalaparse/unit/FailureTests.scala @@ -93,7 +93,7 @@ object FailureTests extends TestSuite{ | } |} """.stripMargin, - aggregate = """("=>" | `:` | "." | TypeArgs | ArgList | `_` | Id | `=` | MatchAscriptionSuffix | &"}")""", + aggregate = """("=>" | `:` | "." | TypeArgs | ArgList | `_` | Id | `=` | MatchAscriptionSuffix | "}")""", terminals = null, found ="1\n" ) @@ -529,7 +529,7 @@ object FailureTests extends TestSuite{ | { a: L = } |} """.stripMargin, - aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | "=>" | BlockLambda | BlockStat | &"}")""", + aggregate = """("." | TypeArgs | `#` | Annot | `with` | { | `*` | Id | "=>" | BlockLambda | BlockStat | "}")""", terminals = null, found = "= }" ) @@ -669,7 +669,7 @@ object FailureTests extends TestSuite{ | } |} """.stripMargin, - aggregate = """(BlockLambda | BlockStat | &"}")""", + aggregate = """(BlockLambda | BlockStat | "}")""", terminals = null, found = "case for" ) @@ -978,7 +978,7 @@ object FailureTests extends TestSuite{ | val x = 1 | ; | """.stripMargin, - aggregate = """(BlockLambda | BlockStat | &"}")""", + aggregate = """(BlockLambda | BlockStat | "}")""", terminals = null, found = "" ) From a3eda3adf2df710637fe5dcb34d0d7ddc2838621 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 7 Mar 2023 14:15:35 +0800 Subject: [PATCH 48/49] . --- fastparse/src/fastparse/SharedPackageDefs.scala | 8 +++++++- fastparse/test/src/fastparse/FailureTests.scala | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 616a8cba..5e8c4807 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -274,8 +274,14 @@ object SharedPackageDefs{ else ctx.freshSuccessUnit(startPos) if (ctx.verboseFailures) { + // Unlike most other data on `ctx`, `terminalParserMsgs` is normally + // append-only. Thus when we're inside the unary_! expression, it + // continually appends to `terminalParserMsgs` sub-parsers that could + // have succeeded within it, but are irrelevant to the user because + // we *want* the contents of the unary_! to fail! Thus, we reset + // `terminalParserMsgs` once we exit the unary_!, to ensure these do not + // end up in error messages ctx.terminalParserMsgs = startTerminals - ctx.aggregateParserMsgs = Msgs.empty ctx.reportTerminalMsg(startPos, Msgs.empty) } res.cut = startCut diff --git a/fastparse/test/src/fastparse/FailureTests.scala b/fastparse/test/src/fastparse/FailureTests.scala index 1dca56a9..4099a477 100644 --- a/fastparse/test/src/fastparse/FailureTests.scala +++ b/fastparse/test/src/fastparse/FailureTests.scala @@ -301,6 +301,9 @@ object FailureTests extends TestSuite{ parse(_) } ) + // Commented out for now, until we can figure out a better story + // around the error reporting of negative lookaheads + // test("negative") - checkOffset( // input = "5", // expected = """![0-6]""", From ed432235da16e2961c12afdcf0c7e43d89ec5160 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 7 Mar 2023 14:25:04 +0800 Subject: [PATCH 49/49] . --- fastparse/src-2/fastparse/package.scala | 53 ++++++++++++++++++- fastparse/src-3/fastparse/package.scala | 50 ++++++++++++++++- .../src/fastparse/SharedPackageDefs.scala | 52 ------------------ 3 files changed, 101 insertions(+), 54 deletions(-) diff --git a/fastparse/src-2/fastparse/package.scala b/fastparse/src-2/fastparse/package.scala index 4de06306..62d7e5fd 100644 --- a/fastparse/src-2/fastparse/package.scala +++ b/fastparse/src-2/fastparse/package.scala @@ -276,9 +276,60 @@ package object fastparse extends fastparse.SharedPackageDefs { * fails if the wrapped parser succeeds. In all cases, it ends up * consuming zero characters. */ - def unary_!(implicit ctx: P[Any]) : P[Unit] = SharedPackageDefs.unary_!(parse0) + def unary_!(implicit ctx: P[Any]) : P[Unit] = { + val startPos = ctx.index + val startCut = ctx.cut + val oldNoCut = ctx.noDropBuffer + ctx.noDropBuffer = true + val startTerminals = ctx.terminalParserMsgs + parse0() + ctx.noDropBuffer = oldNoCut + val msg = ctx.shortParserMsg + + val res = + if (ctx.isSuccess) ctx.freshFailure(startPos) + else ctx.freshSuccessUnit(startPos) + + if (ctx.verboseFailures) { + // Unlike most other data on `ctx`, `terminalParserMsgs` is normally + // append-only. Thus when we're inside the unary_! expression, it + // continually appends to `terminalParserMsgs` sub-parsers that could + // have succeeded within it, but are irrelevant to the user because + // we *want* the contents of the unary_! to fail! Thus, we reset + // `terminalParserMsgs` once we exit the unary_!, to ensure these do not + // end up in error messages + ctx.terminalParserMsgs = startTerminals + ctx.reportTerminalMsg(startPos, Msgs.empty) + } + res.cut = startCut + res + } } + + /** + * Positive lookahead operator: succeeds if the wrapped parser succeeds and + * fails if the wrapped parser fails, but in all cases consumes zero + * characters. + */ + def &(parse: => P[_])(implicit ctx: P[_]): P[Unit] = { + + val startPos = ctx.index + val startCut = ctx.cut + val oldNoCut = ctx.noDropBuffer + ctx.noDropBuffer = true + parse + ctx.noDropBuffer = oldNoCut + val msg = ctx.shortParserMsg + + val res = + if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) + else ctx.asInstanceOf[P[Unit]] + + res.cut = startCut + res + } + /** * Provides logging-related [[LogByNameOps]] implicits on [[String]]. */ diff --git a/fastparse/src-3/fastparse/package.scala b/fastparse/src-3/fastparse/package.scala index aec2fc87..befc1b96 100644 --- a/fastparse/src-3/fastparse/package.scala +++ b/fastparse/src-3/fastparse/package.scala @@ -242,8 +242,56 @@ package object fastparse extends fastparse.SharedPackageDefs { * fails if the wrapped parser succeeds. In all cases, it ends up * consuming zero characters. */ - def unary_!(implicit ctx: P[Any]): P[Unit] = SharedPackageDefs.unary_!(() => parse0) + inline def unary_!(implicit ctx: P[Any]): P[Unit] = { + + val startPos = ctx.index + val startCut = ctx.cut + val oldNoCut = ctx.noDropBuffer + ctx.noDropBuffer = true + val startTerminals = ctx.terminalParserMsgs + parse0 + ctx.noDropBuffer = oldNoCut + + val res = + if (ctx.isSuccess) ctx.freshFailure(startPos) + else ctx.freshSuccessUnit(startPos) + + if (ctx.verboseFailures) { + // Unlike most other data on `ctx`, `terminalParserMsgs` is normally + // append-only. Thus when we're inside the unary_! expression, it + // continually appends to `terminalParserMsgs` sub-parsers that could + // have succeeded within it, but are irrelevant to the user because + // we *want* the contents of the unary_! to fail! Thus, we reset + // `terminalParserMsgs` once we exit the unary_!, to ensure these do not + // end up in error messages + ctx.terminalParserMsgs = startTerminals + ctx.reportTerminalMsg(startPos, Msgs.empty) + } + res.cut = startCut + res + } + + /** + * Positive lookahead operator: succeeds if the wrapped parser succeeds and + * fails if the wrapped parser fails, but in all cases consumes zero + * characters. + */ + inline def &(inline parse: => P[_])(implicit ctx: P[_]): P[Unit] = { + val startPos = ctx.index + val startCut = ctx.cut + val oldNoCut = ctx.noDropBuffer + ctx.noDropBuffer = true + parse + ctx.noDropBuffer = oldNoCut + + val res = + if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) + else ctx.asInstanceOf[P[Unit]] + + res.cut = startCut + res + } /** Provides logging-related [[LogByNameOps]] implicits on [[String]]. */ implicit def LogOpsStr(parse0: String)(implicit ctx: P[Any]): fastparse.LogByNameOps[Unit] = LogByNameOps(parse0) diff --git a/fastparse/src/fastparse/SharedPackageDefs.scala b/fastparse/src/fastparse/SharedPackageDefs.scala index 5e8c4807..76201a01 100644 --- a/fastparse/src/fastparse/SharedPackageDefs.scala +++ b/fastparse/src/fastparse/SharedPackageDefs.scala @@ -86,28 +86,6 @@ trait SharedPackageDefs { res } - /** - * Positive lookahead operator: succeeds if the wrapped parser succeeds and - * fails if the wrapped parser fails, but in all cases consumes zero - * characters. - */ - def &(parse: => P[_])(implicit ctx: P[_]): P[Unit] = { - - val startPos = ctx.index - val startCut = ctx.cut - val oldNoCut = ctx.noDropBuffer - ctx.noDropBuffer = true - parse - ctx.noDropBuffer = oldNoCut - val msg = ctx.shortParserMsg - - val res = - if (ctx.isSuccess) ctx.freshSuccessUnit(startPos) - else ctx.asInstanceOf[P[Unit]] - - res.cut = startCut - res - } /** * Parser that is only successful at the end of the input. Useful to ensure @@ -259,36 +237,6 @@ object SharedPackageDefs{ res2.asInstanceOf[P[T]] } - def unary_!(parse0: () => P[_])(implicit ctx: P[Any]): P[Unit] = { - val startPos = ctx.index - val startCut = ctx.cut - val oldNoCut = ctx.noDropBuffer - ctx.noDropBuffer = true - val startTerminals = ctx.terminalParserMsgs - parse0() - ctx.noDropBuffer = oldNoCut - val msg = ctx.shortParserMsg - - val res = - if (ctx.isSuccess) ctx.freshFailure(startPos) - else ctx.freshSuccessUnit(startPos) - - if (ctx.verboseFailures) { - // Unlike most other data on `ctx`, `terminalParserMsgs` is normally - // append-only. Thus when we're inside the unary_! expression, it - // continually appends to `terminalParserMsgs` sub-parsers that could - // have succeeded within it, but are irrelevant to the user because - // we *want* the contents of the unary_! to fail! Thus, we reset - // `terminalParserMsgs` once we exit the unary_!, to ensure these do not - // end up in error messages - ctx.terminalParserMsgs = startTerminals - ctx.reportTerminalMsg(startPos, Msgs.empty) - } - res.cut = startCut - res - } - - /** Wraps a parser to log when it succeeds and fails, and at what index. * Useful for seeing what is going on within your parser. Nicely indents * the logs for easy reading