diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d index 325f533f..168943af 100644 --- a/src/dparse/lexer.d +++ b/src/dparse/lexer.d @@ -167,12 +167,53 @@ mixin template TokenTriviaFields() int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc { return opCmp(other.index); } + + string toString() const @safe pure + { + import std.array : appender; + + auto sink = appender!string; + toString(sink); + return sink.data; + } + + void toString(R)(auto ref R sink) const + { + import std.conv : to; + import dparse.lexer : str; + + sink.put("tok!\""); + sink.put(str(type)); + sink.put("\"("); + sink.put("text: "); + sink.put([text].to!string[1 .. $ - 1]); // escape hack + sink.put(", index: "); + sink.put(index.to!string); + sink.put(", line: "); + sink.put(line.to!string); + sink.put(", column: "); + sink.put(column.to!string); + sink.put(", trivia: { leading: ["); + foreach (i, tok; leadingTrivia) + { + if (i != 0) sink.put(", "); + tok.toString(sink); + } + sink.put("], trailing: ["); + foreach (i, tok; trailingTrivia) + { + if (i != 0) sink.put(", "); + tok.toString(sink); + } + sink.put("]}"); + sink.put(")"); + } } // mixin in from dparse.lexer to make error messages more managable size as the // entire string is dumped when there is a type mismatch. private immutable extraFields = "import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;"; -private immutable extraFieldsBare = " +private immutable extraFieldsBare = q{ import dparse.lexer : Token; this(Token token) pure nothrow @safe @nogc { @@ -188,7 +229,35 @@ private immutable extraFieldsBare = " int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc { return opCmp(other.index); } -"; + + string toString() const @safe pure + { + import std.array : appender; + + auto sink = appender!string; + toString(sink); + return sink.data; + } + + void toString(R)(auto ref R sink) const + { + import std.conv : to; + import dparse.lexer : str; + + sink.put("trivia!\""); + sink.put(str(type)); + sink.put("\"("); + sink.put("text: "); + sink.put([text].to!string[1 .. $ - 1]); // escape hack + sink.put(", index: "); + sink.put(index.to!string); + sink.put(", line: "); + sink.put(line.to!string); + sink.put(", column: "); + sink.put(column.to!string); + sink.put(")"); + } +}; /// The token type in the D lexer public alias Token = std.experimental.lexer.TokenStructure!(IdType, extraFields); @@ -486,7 +555,7 @@ public bool isLiteral(IdType type) pure nothrow @safe @nogc * `leadingTrivia` until there is the EOF, where it will be attached as * `trailingTrivia` again. */ -const(Token)[] getTokensForParser(R)(R sourceCode, LexerConfig config, StringCache* cache) +Token[] getTokensForParser(R)(R sourceCode, LexerConfig config, StringCache* cache) if (is(Unqual!(ElementEncodingType!R) : ubyte) && isDynamicArray!R) { config.whitespaceBehavior = WhitespaceBehavior.include;