Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RFC] Case Insensitivity Proof of Concept #1092

Closed
wants to merge 9 commits into from
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,5 @@ Errors() ::= ""
grammar(grammarName) ::= <<
lexer grammar <grammarName>;
DASHBRACK : [\\-\]]+ {<writeln("\"DASHBRACK\"")>} ;
WS : [ \u]+ -> skip ;
WS : [ ]+ -> skip ;
>>
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ Errors() ::= ""
grammar(grammarName) ::= <<
lexer grammar <grammarName>;
I : [0-9]+ {<writeln("\"I\"")>} ;
WS : [ \u]+ -> skip ;
WS : [ ]+ -> skip ;
>>

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ TestTemplates ::= [
"CharSetWithMissingEndRange": [],
"CharSetWithMissingEscapeChar": [],
"CharSetWithEscapedChar": [],
"CharSetWithReversedRange": [],
"CharSetWithQuote1": [],
"CharSetWithQuote2": [],
"PositionAdjustingLexer": [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,8 @@ public boolean createProject() {
XPathExpression exp = XPathFactory.newInstance().newXPath()
.compile("/Project/ItemGroup/ProjectReference[@Include='" + runtimeName + "']");
Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE);
if (isWindows() && runtimeProjPath.startsWith("/"))
runtimeProjPath = runtimeProjPath.substring(1);
node.setAttribute("Include", runtimeProjPath.replace("/", "\\"));
// update project file list
exp = XPathFactory.newInstance().newXPath().compile("/Project/ItemGroup[Compile/@Include='AssemblyInfo.cs']");
Expand Down Expand Up @@ -533,7 +535,8 @@ public String execTest() {
String exec = locateExec();
String[] args = isWindows() ?
new String[] { exec, new File(tmpdir, "input").getAbsolutePath() } :
new String[] { "mono", exec, new File(tmpdir, "input").getAbsolutePath() };
// inline disabling to avoid "Method is too complex" exception.
new String[] { "mono", "-O=-inline", exec, new File(tmpdir, "input").getAbsolutePath() };
ProcessBuilder pb = new ProcessBuilder(args);
pb.directory(new File(tmpdir));
Process process = pb.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public void testCharSetWithEscapedChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(94);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("DASHBRACK : [\\-\\]]+ {Console.WriteLine(\"DASHBRACK\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="- ] ";
String found = execLexer("L.g4", grammar, "L", input, false);
Expand Down Expand Up @@ -197,7 +197,7 @@ public void testCharSetWithMissingEscapeChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(77);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("I : [0-9]+ {Console.WriteLine(\"I\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="34 ";
String found = execLexer("L.g4", grammar, "L", input, false);
Expand Down Expand Up @@ -245,25 +245,6 @@ public void testCharSetWithQuote2() throws Exception {
"[@1,4:3='<EOF>',<-1>,1:4]\n", found);
assertNull(this.stderrDuringParse);

}
/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testCharSetWithReversedRange() throws Exception {
mkdir(tmpdir);

StringBuilder grammarBuilder = new StringBuilder(78);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("A : [z-a9]+ {Console.WriteLine(\"A\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="9";
String found = execLexer("L.g4", grammar, "L", input, false);
assertEquals(
"A\n" +
"[@0,0:0='9',<1>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n", found);
assertNull(this.stderrDuringParse);

}
/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public void testCharSetWithEscapedChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(95);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("DASHBRACK : [\\-\\]]+ {System.out.println(\"DASHBRACK\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="- ] ";
Expand Down Expand Up @@ -214,7 +214,7 @@ public void testCharSetWithMissingEscapeChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(78);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("I : [0-9]+ {System.out.println(\"I\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="34 ";
Expand Down Expand Up @@ -269,27 +269,6 @@ public void testCharSetWithQuote2() throws Exception {

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testCharSetWithReversedRange() throws Exception {
mkdir(tmpdir);

StringBuilder grammarBuilder = new StringBuilder(79);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("A : [z-a9]+ {System.out.println(\"A\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="9";
String found = execLexer("L.g4", grammar, "L", input, false);
assertEquals(
"A\n" +
"[@0,0:0='9',<1>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n", found);
assertNull(this.stderrDuringParse);

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testEOFByItself() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ public void testCharSetWithEscapedChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(88);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("DASHBRACK : [\\-\\]]+ {console.log(\"DASHBRACK\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="- ] ";
String found = execLexer("L.g4", grammar, "L", input, false);
Expand Down Expand Up @@ -199,7 +199,7 @@ public void testCharSetWithMissingEscapeChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(71);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("I : [0-9]+ {console.log(\"I\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="34 ";
String found = execLexer("L.g4", grammar, "L", input, false);
Expand Down Expand Up @@ -247,25 +247,6 @@ public void testCharSetWithQuote2() throws Exception {
"[@1,4:3='<EOF>',<-1>,1:4]\n", found);
assertNull(this.stderrDuringParse);

}
/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testCharSetWithReversedRange() throws Exception {
mkdir(tmpdir);

StringBuilder grammarBuilder = new StringBuilder(72);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("A : [z-a9]+ {console.log(\"A\");} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
String grammar = grammarBuilder.toString();
String input ="9";
String found = execLexer("L.g4", grammar, "L", input, false);
assertEquals(
"A\n" +
"[@0,0:0='9',<1>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n", found);
assertNull(this.stderrDuringParse);

}
/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public void testCharSetWithEscapedChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(81);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("DASHBRACK : [\\-\\]]+ {print(\"DASHBRACK\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="- ] ";
Expand Down Expand Up @@ -214,7 +214,7 @@ public void testCharSetWithMissingEscapeChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(64);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("I : [0-9]+ {print(\"I\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="34 ";
Expand Down Expand Up @@ -269,27 +269,6 @@ public void testCharSetWithQuote2() throws Exception {

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testCharSetWithReversedRange() throws Exception {
mkdir(tmpdir);

StringBuilder grammarBuilder = new StringBuilder(65);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("A : [z-a9]+ {print(\"A\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="9";
String found = execLexer("L.g4", grammar, "L", input, false);
assertEquals(
"A\n" +
"[@0,0:0='9',<1>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n", found);
assertNull(this.stderrDuringParse);

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testEOFByItself() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public void testCharSetWithEscapedChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(81);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("DASHBRACK : [\\-\\]]+ {print(\"DASHBRACK\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="- ] ";
Expand Down Expand Up @@ -214,7 +214,7 @@ public void testCharSetWithMissingEscapeChar() throws Exception {
StringBuilder grammarBuilder = new StringBuilder(64);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("I : [0-9]+ {print(\"I\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
grammarBuilder.append("WS : [ ]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="34 ";
Expand Down Expand Up @@ -269,27 +269,6 @@ public void testCharSetWithQuote2() throws Exception {

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testCharSetWithReversedRange() throws Exception {
mkdir(tmpdir);

StringBuilder grammarBuilder = new StringBuilder(65);
grammarBuilder.append("lexer grammar L;\n");
grammarBuilder.append("A : [z-a9]+ {print(\"A\")} ;\n");
grammarBuilder.append("WS : [ \\u]+ -> skip ;");
String grammar = grammarBuilder.toString();

String input ="9";
String found = execLexer("L.g4", grammar, "L", input, false);
assertEquals(
"A\n" +
"[@0,0:0='9',<1>,1:0]\n" +
"[@1,1:0='<EOF>',<-1>,1:1]\n", found);
assertNull(this.stderrDuringParse);

}

/* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */
@Test
public void testEOFByItself() throws Exception {
Expand Down
52 changes: 26 additions & 26 deletions tool-testsuite/test/org/antlr/v4/test/tool/TestATNConstruction.java
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,14 @@ public void testA() throws Exception {
"A : ('0x' | '0X') ;");
String expecting =
"s0->RuleStart_A_1\n" +
"RuleStart_A_1->BlockStart_7\n" +
"BlockStart_7->s3\n" +
"BlockStart_7->s5\n" +
"s3-'0'->s4\n" +
"s5-'0'->s6\n" +
"s4-'x'->BlockEnd_8\n" +
"s6-'X'->BlockEnd_8\n" +
"BlockEnd_8->RuleStop_A_2\n";
"RuleStart_A_1->BlockStart_11\n" +
"BlockStart_11->s3\n" +
"BlockStart_11->s7\n" +
"s3-'0'->s5\n" +
"s7-'0'->s9\n" +
"s5-'x'->BlockEnd_12\n" +
"s9-'X'->BlockEnd_12\n" +
"BlockEnd_12->RuleStop_A_2\n";
checkTokensRule(g, null, expecting);
}
@Test public void testRange() throws Exception {
Expand All @@ -140,14 +140,14 @@ public void testA() throws Exception {
);
String expecting =
"s0->RuleStart_A_1\n" +
"RuleStart_A_1->BlockStart_7\n" +
"BlockStart_7->s3\n" +
"BlockStart_7->s5\n" +
"RuleStart_A_1->BlockStart_9\n" +
"BlockStart_9->s3\n" +
"BlockStart_9->s6\n" +
"s3-'a'..'c'->s4\n" +
"s5-'q'->s6\n" +
"s4-'h'->BlockEnd_8\n" +
"s6-'j'..'l'->BlockEnd_8\n" +
"BlockEnd_8->RuleStop_A_2\n";
"s6-'q'->s8\n" +
"s4-'h'->BlockEnd_10\n" +
"s8-'j'..'l'->BlockEnd_10\n" +
"BlockEnd_10->RuleStop_A_2\n";
checkTokensRule(g, null, expecting);
}
@Test public void testStringLiteralInParser() throws Exception {
Expand Down Expand Up @@ -927,11 +927,11 @@ public void testA() throws Exception {
"s0->RuleStart_A_2\n" +
"s0->RuleStart_X_4\n" +
"RuleStart_A_2->s10\n" +
"RuleStart_X_4->s12\n" +
"s10-'a'->s11\n" +
"s12-'x'->s13\n" +
"s11->RuleStop_A_3\n" +
"s13->RuleStop_X_5\n";
"RuleStart_X_4->s13\n" +
"s10-'a'->s12\n" +
"s13-'x'->s15\n" +
"s12->RuleStop_A_3\n" +
"s15->RuleStop_X_5\n";
checkTokensRule(g, "DEFAULT_MODE", expecting);
}
@Test public void testMode() throws Exception {
Expand All @@ -945,12 +945,12 @@ public void testA() throws Exception {
String expecting =
"s1->RuleStart_B_6\n" +
"s1->RuleStart_C_8\n" +
"RuleStart_B_6->s14\n" +
"RuleStart_C_8->s16\n" +
"s14-'b'->s15\n" +
"s16-'c'->s17\n" +
"s15->RuleStop_B_7\n" +
"s17->RuleStop_C_9\n";
"RuleStart_B_6->s16\n" +
"RuleStart_C_8->s19\n" +
"s16-'b'->s18\n" +
"s19-'c'->s21\n" +
"s18->RuleStop_B_7\n" +
"s21->RuleStop_C_9\n";
checkTokensRule(g, "FOO", expecting);
}
void checkTokensRule(LexerGrammar g, String modeName, String expecting) {
Expand Down
Loading