From d3767e030375f4526ab1966dbcf71c72a6bb27b9 Mon Sep 17 00:00:00 2001
From: WebFreak001 <gh@webfreak.org>
Date: Sun, 29 Oct 2023 07:32:01 +0100
Subject: [PATCH 1/2] add Token.toString implementation

for neater debug printing
---
 src/dparse/lexer.d | 73 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 71 insertions(+), 2 deletions(-)

diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d
index 325f533f..531ae887 100644
--- a/src/dparse/lexer.d
+++ b/src/dparse/lexer.d
@@ -167,12 +167,53 @@ mixin template TokenTriviaFields()
     int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc {
         return opCmp(other.index);
     }
+
+    string toString() const @safe pure
+    {
+        import std.array : appender;
+
+        auto sink = appender!string;
+        toString(sink);
+        return sink.data;
+    }
+
+    void toString(R)(auto ref R sink) const
+    {
+        import std.conv : to;
+        import dparse.lexer : str;
+
+        sink.put("tok!\"");
+        sink.put(str(type));
+        sink.put("\"(");
+        sink.put("text: ");
+        sink.put([text].to!string[1 .. $ - 1]); // escape hack
+        sink.put(", index: ");
+        sink.put(index.to!string);
+        sink.put(", line: ");
+        sink.put(line.to!string);
+        sink.put(", column: ");
+        sink.put(column.to!string);
+        sink.put(", trivia: { leading: [");
+        foreach (i, tok; leadingTrivia)
+        {
+            if (i != 0) sink.put(", ");
+            tok.toString(sink);
+        }
+        sink.put("], trailing: [");
+        foreach (i, tok; trailingTrivia)
+        {
+            if (i != 0) sink.put(", ");
+            tok.toString(sink);
+        }
+        sink.put("]}");
+        sink.put(")");
+    }
 }
 
 // mixin in from dparse.lexer to make error messages more managable size as the
 // entire string is dumped when there is a type mismatch.
 private immutable extraFields = "import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;";
-private immutable extraFieldsBare = "
+private immutable extraFieldsBare = q{
     import dparse.lexer : Token;
 
     this(Token token) pure nothrow @safe @nogc {
@@ -188,7 +229,35 @@ private immutable extraFieldsBare = "
     int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc {
         return opCmp(other.index);
     }
-";
+
+    string toString() const @safe pure
+    {
+        import std.array : appender;
+
+        auto sink = appender!string;
+        toString(sink);
+        return sink.data;
+    }
+
+    void toString(R)(auto ref R sink) const
+    {
+        import std.conv : to;
+        import dparse.lexer : str;
+
+        sink.put("trivia!\"");
+        sink.put(str(type));
+        sink.put("\"(");
+        sink.put("text: ");
+        sink.put([text].to!string[1 .. $ - 1]); // escape hack
+        sink.put(", index: ");
+        sink.put(index.to!string);
+        sink.put(", line: ");
+        sink.put(line.to!string);
+        sink.put(", column: ");
+        sink.put(column.to!string);
+        sink.put(")");
+    }
+};
 
 /// The token type in the D lexer
 public alias Token = std.experimental.lexer.TokenStructure!(IdType, extraFields);

From 8588b70b487b034899b29f7649b5b6eb4cd748d4 Mon Sep 17 00:00:00 2001
From: WebFreak001 <gh@webfreak.org>
Date: Sun, 29 Oct 2023 07:34:04 +0100
Subject: [PATCH 2/2] make getTokensForParser return non-const

since data is allocated with an appender that is not cleared, GC owns
the memory and may be modified by the user if they wish to.

Can still be used in all places where const was used before.
---
 src/dparse/lexer.d | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d
index 531ae887..168943af 100644
--- a/src/dparse/lexer.d
+++ b/src/dparse/lexer.d
@@ -555,7 +555,7 @@ public bool isLiteral(IdType type) pure nothrow @safe @nogc
  * `leadingTrivia` until there is the EOF, where it will be attached as
  * `trailingTrivia` again.
  */
-const(Token)[] getTokensForParser(R)(R sourceCode, LexerConfig config, StringCache* cache)
+Token[] getTokensForParser(R)(R sourceCode, LexerConfig config, StringCache* cache)
 if (is(Unqual!(ElementEncodingType!R) : ubyte) && isDynamicArray!R)
 {
     config.whitespaceBehavior = WhitespaceBehavior.include;