Merge pull request #465 from b3b00/bugfix/issue464

Bugfix/issue464

src/sly/lexer/EOLType.cs

@@ -8,6 +8,8 @@ public enum EolType
         Mac,
         Environment,
 
+        Eof,
+
         No
     }
 }

src/sly/lexer/fsm/EOLManager.cs

@@ -18,7 +18,7 @@ public static ReadOnlyMemory<char> GetToEndOfLine(this ReadOnlyMemory<char> valu
                 end = IsEndOfLine(value, CurrentPosition);
             }
 
-            return value.Slice(position, CurrentPosition - position + (end == EolType.Windows ? 2 : 1));
+            return value.Slice(position, CurrentPosition - position + (end == EolType.Eof ? 0 : end == EolType.Windows ? 2 : 1));
         }
 
         public static ReadOnlyMemory<char> RemoveEndOfLineChars(this ReadOnlyMemory<char> value)
@@ -38,6 +38,10 @@ public static ReadOnlyMemory<char> RemoveEndOfLineChars(this ReadOnlyMemory<char
         public static EolType IsEndOfLine(ReadOnlyMemory<char> value, int position)
         {
             var end = EolType.No;
+            if (position >= value.Length)
+            {
+                return EolType.Eof;
+            }
             var n = value.At<char>(position);
             switch (n)
             {

tests/ParserTests/comments/SingleLineCommentsTest.cs

@@ -40,6 +40,27 @@ public void TestGenericMultiLineCommentWithSingleLineComment()
             var tokens = r.Tokens;
             Check.That(r.Error.UnexpectedChar).IsEqualTo('*');
         }
+
+        [Fact]
+        public void TestCommentOnLastLine()
+        {
+            var lexerRes = LexerBuilder.BuildLexer(new BuildResult<ILexer<SingleLineCommentsToken>>());
+            Check.That(lexerRes).IsOk();
+
+            var lexer = lexerRes.Result as GenericLexer<SingleLineCommentsToken>;
+
+            var dump = lexer.ToString();
+
+            var r =lexer?.Tokenize(@"
+1.0
+// this is the final comment");
+            Check.That(r).IsOkLexing();
+            var comment = r.Tokens[1];
+            Check.That(comment).IsNotNull();
+            Check.That(comment.TokenID).IsEqualTo(SingleLineCommentsToken.COMMENT);
+            Check.That(comment.Value).Contains("this is the final comment");
+
+        }
 
         [Fact]
         public void TestGenericSingleLineComment()