Fix to have parser test for EOI as last token

README.md

@@ -50,11 +50,14 @@ represent (prefix or infix) parselets. Instead, it relies on them being simple
 functions:
 
 ```c#
+// version 2.0+
+
 public static TResult
     Parse<TState, TKind, TToken, TPrecedence, TResult>(
         TState state,
         TPrecedence initialPrecedence, IComparer<TPrecedence> precedenceComparer,
         IEqualityComparer<TKind> kindEqualityComparer,
+        TKind eoi, Func<TToken, Exception> eoiErrorSelector,
         Func<TKind, TToken, TState, Func<TToken, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>> prefixFunction,
         Func<TKind, TToken, TState, (TPrecedence, Func<TToken, TResult, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>)?> infixFunction,
         IEnumerable<(TKind, TToken)> lexer)
@@ -67,6 +70,9 @@ parameter in order:
 - the initial precedence (this is usually 0 if `TPrecedence` is an `int`)
 - a precedence comparer
 - an equality comparer that can compare two token kinds
+- a token kind marking _end of input_ (EOI)
+- a function used to project an `Exception` when the EOI token is not the
+  the last token of the tokens sequence
 - a prefix function
 - an infix function
 - a sequence of token kind and token pairs (2-tuple) yielded by a lexer

src/Parser.cs

@@ -32,6 +32,11 @@ static partial class Parser
         /// <typeparam name="TPrecedence">The type of precedence.</typeparam>
         /// <typeparam name="TResult">The type of the result.</typeparam>
         /// <param name="initialPrecedence">The initial precedence.</param>
+        /// <param name="eoi">A token kind that marks the end of input.</param>
+        /// <param name="eoiErrorSelector">
+        /// A function that projects an <see cref="Exception"/> when the
+        /// end-of-input token is not the the last token of the
+        /// <paramref name="tokens"/> sequence.</param>
         /// <param name="prefixSelector">
         /// A function that maps a token to a prefix parser function. If the
         /// given token is not a prefix then the function must fail by throwing
@@ -55,10 +60,11 @@ static partial class Parser
         public static TResult
             Parse<TKind, TToken, TPrecedence, TResult>(
                 TPrecedence initialPrecedence,
+                TKind eoi, Func<TToken, Exception> eoiErrorSelector,
                 Func<TKind, TToken, Func<TToken, Parser<Unit, TKind, TToken, TPrecedence, TResult>, TResult>> prefixSelector,
                 Func<TKind, TToken, (TPrecedence, Func<TToken, TResult, Parser<Unit, TKind, TToken, TPrecedence, TResult>, TResult>)?> infixSelector,
                 IEnumerable<(TKind, TToken)> tokens) =>
-            Parse(default(Unit), initialPrecedence,
+            Parse(default(Unit), initialPrecedence, eoi, eoiErrorSelector,
                   (k, t, s) => prefixSelector(k, t), (k, t, s) => infixSelector(k, t), tokens);
 
         /// <summary>
@@ -77,6 +83,11 @@ public static TResult
         /// <param name="kindEqualityComparer">
         /// An <see cref="IEqualityComparer{T}"/> to use to compare one token
         /// kind with another for equality.</param>
+        /// <param name="eoi">A token kind that marks the end of input.</param>
+        /// <param name="eoiErrorSelector">
+        /// A function that projects an <see cref="Exception"/> when the
+        /// end-of-input token is not the the last token of the
+        /// <paramref name="tokens"/> sequence.</param>
         /// <param name="prefixSelector">
         /// A function that maps a token to a prefix parser function. If the
         /// given token is not a prefix then the function must fail by throwing
@@ -101,10 +112,12 @@ public static TResult
             Parse<TKind, TToken, TPrecedence, TResult>(
                 TPrecedence initialPrecedence, IComparer<TPrecedence> precedenceComparer,
                 IEqualityComparer<TKind> kindEqualityComparer,
+                TKind eoi, Func<TToken, Exception> eoiErrorSelector,
                 Func<TKind, TToken, Func<TToken, Parser<Unit, TKind, TToken, TPrecedence, TResult>, TResult>> prefixSelector,
                 Func<TKind, TToken, (TPrecedence, Func<TToken, TResult, Parser<Unit, TKind, TToken, TPrecedence, TResult>, TResult>)?> infixSelector,
                 IEnumerable<(TKind, TToken)> tokens) =>
             Parse(default(Unit), initialPrecedence, precedenceComparer, kindEqualityComparer,
+                  eoi, eoiErrorSelector,
                   (k, t, s) => prefixSelector(k, t), (k, t, s) => infixSelector(k, t), tokens);
 
         /// <summary>
@@ -121,6 +134,11 @@ public static TResult
         /// <typeparam name="TResult">The type of the result.</typeparam>
         /// <param name="state">A user-defined state.</param>
         /// <param name="initialPrecedence">The initial precedence.</param>
+        /// <param name="eoi">A token kind that marks the end of input.</param>
+        /// <param name="eoiErrorSelector">
+        /// A function that projects an <see cref="Exception"/> when the
+        /// end-of-input token is not the the last token of the
+        /// <paramref name="tokens"/> sequence.</param>
         /// <param name="prefixSelector">
         /// A function that maps a token to a prefix parser function. If the
         /// given token is not a prefix then the function must fail by throwing
@@ -145,10 +163,12 @@ public static TResult
             Parse<TState, TKind, TToken, TPrecedence, TResult>(
                 TState state,
                 TPrecedence initialPrecedence,
+                TKind eoi, Func<TToken, Exception> eoiErrorSelector,
                 Func<TKind, TToken, TState, Func<TToken, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>> prefixSelector,
                 Func<TKind, TToken, TState, (TPrecedence, Func<TToken, TResult, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>)?> infixSelector,
                 IEnumerable<(TKind, TToken)> tokens) =>
             Parse(state, initialPrecedence, Comparer<TPrecedence>.Default, EqualityComparer<TKind>.Default,
+                  eoi, eoiErrorSelector,
                   prefixSelector, infixSelector, tokens);
 
         /// <summary>
@@ -170,6 +190,11 @@ public static TResult
         /// <param name="kindEqualityComparer">
         /// An <see cref="IEqualityComparer{T}"/> to use to compare one token
         /// kind with another for equality.</param>
+        /// <param name="eoi">A token kind that marks the end of input.</param>
+        /// <param name="eoiErrorSelector">
+        /// A function that projects an <see cref="Exception"/> when the
+        /// end-of-input token is not the the last token of the
+        /// <paramref name="tokens"/> sequence.</param>
         /// <param name="prefixSelector">
         /// A function that maps a token to a prefix parser function. If the
         /// given token is not a prefix then the function must fail by throwing
@@ -189,12 +214,12 @@ public static TResult
         /// must represent the end-of-input otherwise the behavior of this
         /// function, and by extension parsing, is undefined.
         /// </remarks>
-
         public static TResult
             Parse<TState, TKind, TToken, TPrecedence, TResult>(
                 TState state,
                 TPrecedence initialPrecedence, IComparer<TPrecedence> precedenceComparer,
                 IEqualityComparer<TKind> kindEqualityComparer,
+                TKind eoi, Func<TToken, Exception> eoiErrorSelector,
                 Func<TKind, TToken, TState, Func<TToken, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>> prefixSelector,
                 Func<TKind, TToken, TState, (TPrecedence, Func<TToken, TResult, Parser<TState, TKind, TToken, TPrecedence, TResult>, TResult>)?> infixSelector,
                 IEnumerable<(TKind, TToken)> tokens)
@@ -206,7 +231,9 @@ public static TResult
                                                                         kindEqualityComparer,
                                                                         prefixSelector, infixSelector,
                                                                         e);
-            return parser.Parse(initialPrecedence);
+            var result = parser.Parse(initialPrecedence);
+            parser.Read(eoi, (TKind _, (TKind, TToken Token) a) => eoiErrorSelector(a.Token));
+            return result;
         }
     }
 

tests/Bantam.cs

@@ -449,9 +449,9 @@ public ParseException(string message, Exception inner) : base(message, inner) {}
     static class BantamParser
     {
         public static Expression Parse(string source) =>
-            Parser.Parse(0,
-                (TokenType type, Token _) => Spec.Instance.Prefix(type),
-                (TokenType type, Token _) => Spec.Instance.Infix(type),
+            Parser.Parse(0, Eof, t => new Exception(), 
+                (type, _) => Spec.Instance.Prefix(type),
+                (type, _) => Spec.Instance.Infix(type),
                 from t in Lexer.Lex(source)
                 select (t.Type, t));
 

tests/CSharpPreprocessorExpression.cs

@@ -41,6 +41,7 @@ public static bool Evaluate(string expression, Func<string, bool> symbolPredicat
             Gratt.Parser.Parse(
                 new ParseContext(expression, symbolPredicate),
                 Precedence.Default,
+                TokenKind.Eoi, t => new SyntaxErrorException($"Unexpected <{t.Kind}> token at offset {t.Offset}."),
                 (_, token, __) => Spec.Instance.Prefix(token),
                 (kind, _, __) => Spec.Instance.Infix(kind),
                 from t in Scanner.Scan(expression)
@@ -143,8 +144,11 @@ sealed class Spec : IEnumerable
             },
 
             { TokenKind.Bang, (_, parser) => !parser.Parse(Precedence.Prefix) },
-
-            { TokenKind.AmpersandAmpersand, Precedence.LogicalAnd, (a, rbp, p) => a && p.Parse(rbp) },
+            // NOTE! The "&&" evaluation cannot be short-circuited since the
+            //       right side needs to be parsed to ensure it is syntactically
+            //       correct and the end-of-input is reached by full parsing of
+            //       the input.
+            { TokenKind.AmpersandAmpersand, Precedence.LogicalAnd, (a, rbp, p) => p.Parse(rbp) is {} b && a && b },
             { TokenKind.PipePipe          , Precedence.LogicalOr , (a, b) => a || b },
             { TokenKind.EqualEqual        , Precedence.Relational, (a, b) => a == b },
             { TokenKind.BangEqual         , Precedence.Relational, (a, b) => a != b },