[pre] Fix "Unterminated string literal" for interpolated strings skip…

…ped in conditional compilation. Fix #162
fusionlanguage · May 11, 2024 · feefdcf · feefdcf
1 parent 3aecf51
commit feefdcf
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 24 deletions.
diff --git a/Lexer.fu b/Lexer.fu
@@ -161,7 +161,7 @@ public abstract class FuLexer
 	protected string() StringValue; // for FuToken.LiteralString, FuToken.InterpolatedString, FuToken.Id
 	HashSet<string()>() PreSymbols;
 	bool LineMode = false;
-	bool EnableDocComments = true;
+	bool SkippingUnmet = false;
 	protected bool ParsingTypeArg = false;
 	Stack<bool>() PreElseStack;
 
@@ -453,7 +453,7 @@ public abstract class FuLexer
 
 	protected FuToken ReadString!(bool interpolated)
 	{
-		for (int offset = this.CharOffset; ; ReadCharLiteral()) {
+		for (int offset = this.CharOffset;;) {
 			switch (PeekChar()) {
 			case -1:
 				ReportError("Unterminated string literal");
@@ -473,13 +473,27 @@ public abstract class FuLexer
 				if (interpolated) {
 					int endOffset = this.CharOffset;
 					ReadChar();
-					if (PeekChar() != '{') {
+					if (EatChar('{'))
+						break;
+					if (!this.SkippingUnmet) {
 						this.StringValue = Encoding.UTF8.GetString(this.Input, offset, endOffset - offset);
 						return FuToken.InterpolatedString;
 					}
+					for (;;) {
+						FuToken token = ReadPreToken();
+						if (token == FuToken.RightBrace)
+							break;
+						if (token == FuToken.EndOfFile) {
+							ReportError("Unterminated string literal");
+							return FuToken.EndOfFile;
+						}
+					}
 				}
+				else
+					ReadChar();
 				break;
 			default:
+				ReadCharLiteral();
 				break;
 			}
 		}
@@ -565,7 +579,7 @@ public abstract class FuLexer
 			case '/':
 				if (EatChar('/')) {
 					c = ReadChar();
-					if (c == '/' && this.EnableDocComments) {
+					if (c == '/' && !this.SkippingUnmet) {
 						SkipWhitespace();
 						switch (PeekChar()) {
 						case '\n':
@@ -946,7 +960,7 @@ public abstract class FuLexer
 
 	void SkipUnmet!(FuPreState state)
 	{
-		this.EnableDocComments = false;
+		this.SkippingUnmet = true;
 		for (;;) {
 			// state == FuPreState.NotYet: we are in a conditional that wasn't met yet
 			// else: we are in a conditional that was met before
@@ -989,7 +1003,7 @@ public abstract class FuLexer
 	{
 		for (;;) {
 			// we are in no conditionals or in all met
-			this.EnableDocComments = true;
+			this.SkippingUnmet = false;
 			FuToken token = ReadPreToken();
 			bool matched;
 			switch (token) {

diff --git a/libfut.cpp b/libfut.cpp
@@ -333,7 +333,7 @@ int FuLexer::readCharLiteral()
 
 FuToken FuLexer::readString(bool interpolated)
 {
-	for (int offset = this->charOffset;; readCharLiteral()) {
+	for (int offset = this->charOffset;;) {
 		switch (peekChar()) {
 		case -1:
 			reportError("Unterminated string literal");
@@ -353,13 +353,27 @@ FuToken FuLexer::readString(bool interpolated)
 			if (interpolated) {
 				int endOffset = this->charOffset;
 				readChar();
-				if (peekChar() != '{') {
+				if (eatChar('{'))
+					break;
+				if (!this->skippingUnmet) {
 					this->stringValue = std::string_view(reinterpret_cast<const char *>(this->input + offset), endOffset - offset);
 					return FuToken::interpolatedString;
 				}
+				for (;;) {
+					FuToken token = readPreToken();
+					if (token == FuToken::rightBrace)
+						break;
+					if (token == FuToken::endOfFile) {
+						reportError("Unterminated string literal");
+						return FuToken::endOfFile;
+					}
+				}
 			}
+			else
+				readChar();
 			break;
 		default:
+			readCharLiteral();
 			break;
 		}
 	}
@@ -468,7 +482,7 @@ FuToken FuLexer::readPreToken()
 		case '/':
 			if (eatChar('/')) {
 				c = readChar();
-				if (c == '/' && this->enableDocComments) {
+				if (c == '/' && !this->skippingUnmet) {
 					skipWhitespace();
 					switch (peekChar()) {
 					case '\n':
@@ -1004,7 +1018,7 @@ bool FuLexer::popPreElse(std::string_view directive)
 
 void FuLexer::skipUnmet(FuPreState state)
 {
-	this->enableDocComments = false;
+	this->skippingUnmet = true;
 	for (;;) {
 		switch (readPreToken()) {
 		case FuToken::endOfFile:
@@ -1044,7 +1058,7 @@ void FuLexer::skipUnmet(FuPreState state)
 FuToken FuLexer::readToken()
 {
 	for (;;) {
-		this->enableDocComments = true;
+		this->skippingUnmet = false;
 		FuToken token = readPreToken();
 		bool matched;
 		switch (token) {

diff --git a/libfut.cs b/libfut.cs
@@ -167,7 +167,7 @@ public abstract class FuLexer
 
 		bool LineMode = false;
 
-		bool EnableDocComments = true;
+		bool SkippingUnmet = false;
 
 		protected bool ParsingTypeArg = false;
 
@@ -466,7 +466,7 @@ int ReadCharLiteral()
 
 		protected FuToken ReadString(bool interpolated)
 		{
-			for (int offset = this.CharOffset;; ReadCharLiteral()) {
+			for (int offset = this.CharOffset;;) {
 				switch (PeekChar()) {
 				case -1:
 					ReportError("Unterminated string literal");
@@ -486,13 +486,27 @@ protected FuToken ReadString(bool interpolated)
 					if (interpolated) {
 						int endOffset = this.CharOffset;
 						ReadChar();
-						if (PeekChar() != '{') {
+						if (EatChar('{'))
+							break;
+						if (!this.SkippingUnmet) {
 							this.StringValue = Encoding.UTF8.GetString(this.Input, offset, endOffset - offset);
 							return FuToken.InterpolatedString;
 						}
+						for (;;) {
+							FuToken token = ReadPreToken();
+							if (token == FuToken.RightBrace)
+								break;
+							if (token == FuToken.EndOfFile) {
+								ReportError("Unterminated string literal");
+								return FuToken.EndOfFile;
+							}
+						}
 					}
+					else
+						ReadChar();
 					break;
 				default:
+					ReadCharLiteral();
 					break;
 				}
 			}
@@ -595,7 +609,7 @@ FuToken ReadPreToken()
 				case '/':
 					if (EatChar('/')) {
 						c = ReadChar();
-						if (c == '/' && this.EnableDocComments) {
+						if (c == '/' && !this.SkippingUnmet) {
 							SkipWhitespace();
 							switch (PeekChar()) {
 							case '\n':
@@ -1130,7 +1144,7 @@ bool PopPreElse(string directive)
 
 		void SkipUnmet(FuPreState state)
 		{
-			this.EnableDocComments = false;
+			this.SkippingUnmet = true;
 			for (;;) {
 				switch (ReadPreToken()) {
 				case FuToken.EndOfFile:
@@ -1170,7 +1184,7 @@ void SkipUnmet(FuPreState state)
 		FuToken ReadToken()
 		{
 			for (;;) {
-				this.EnableDocComments = true;
+				this.SkippingUnmet = false;
 				FuToken token = ReadPreToken();
 				bool matched;
 				switch (token) {

diff --git a/libfut.hpp b/libfut.hpp
@@ -522,7 +522,7 @@ class FuLexer
 	int nextChar;
 	std::unordered_set<std::string> preSymbols;
 	bool lineMode = false;
-	bool enableDocComments = true;
+	bool skippingUnmet = false;
 	std::stack<bool> preElseStack;
 	int readByte();
 	static constexpr int replacementChar = 65533;

diff --git a/libfut.js b/libfut.js
@@ -146,7 +146,7 @@ export class FuLexer
 	stringValue;
 	#preSymbols = new Set();
 	#lineMode = false;
-	#enableDocComments = true;
+	#skippingUnmet = false;
 	parsingTypeArg = false;
 	#preElseStack = [];
 
@@ -444,7 +444,7 @@ export class FuLexer
 
 	readString(interpolated)
 	{
-		for (let offset = this.charOffset;; this.#readCharLiteral()) {
+		for (let offset = this.charOffset;;) {
 			switch (this.peekChar()) {
 			case -1:
 				this.reportError("Unterminated string literal");
@@ -464,13 +464,27 @@ export class FuLexer
 				if (interpolated) {
 					let endOffset = this.charOffset;
 					this.readChar();
-					if (this.peekChar() != 123) {
+					if (this.#eatChar(123))
+						break;
+					if (!this.#skippingUnmet) {
 						this.stringValue = new TextDecoder().decode(this.input.subarray(offset, offset + endOffset - offset));
 						return FuToken.INTERPOLATED_STRING;
 					}
+					for (;;) {
+						let token = this.#readPreToken();
+						if (token == FuToken.RIGHT_BRACE)
+							break;
+						if (token == FuToken.END_OF_FILE) {
+							this.reportError("Unterminated string literal");
+							return FuToken.END_OF_FILE;
+						}
+					}
 				}
+				else
+					this.readChar();
 				break;
 			default:
+				this.#readCharLiteral();
 				break;
 			}
 		}
@@ -579,7 +593,7 @@ export class FuLexer
 			case 47:
 				if (this.#eatChar(47)) {
 					c = this.readChar();
-					if (c == 47 && this.#enableDocComments) {
+					if (c == 47 && !this.#skippingUnmet) {
 						this.#skipWhitespace();
 						switch (this.peekChar()) {
 						case 10:
@@ -1119,7 +1133,7 @@ export class FuLexer
 
 	#skipUnmet(state)
 	{
-		this.#enableDocComments = false;
+		this.#skippingUnmet = true;
 		for (;;) {
 			switch (this.#readPreToken()) {
 			case FuToken.END_OF_FILE:
@@ -1159,7 +1173,7 @@ export class FuLexer
 	#readToken()
 	{
 		for (;;) {
-			this.#enableDocComments = true;
+			this.#skippingUnmet = false;
 			let token = this.#readPreToken();
 			let matched;
 			switch (token) {

diff --git a/test/PreStringInterpolated.fu b/test/PreStringInterpolated.fu
@@ -0,0 +1,11 @@
+public static class Test
+{
+	public static bool Run()
+	{
+#if false
+        int a = 0;
+        Console.WriteLine($"{a}");
+#endif
+		return true;
+	}
+}