From 06ff6dcd8774dea3309d5b7de946a46b7c17ef83 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Nov 2022 07:43:58 +0100 Subject: [PATCH 1/3] Relaxed CodeLocationsTest and co. --- .../main/java/org/enso/compiler/TreeToIr.java | 181 +++++++++++--- .../org/enso/interpreter/runtime/Module.java | 5 + .../scala/org/enso/compiler/core/IR.scala | 6 +- .../org/enso/compiler/EnsoCompilerTest.java | 229 +++++++++++++++--- .../org/enso/compiler/ParseStdLibTest.java | 2 +- .../test/CodeLocationsTestInstrument.java | 34 ++- .../interpreter/test/InterpreterTest.scala | 14 +- .../test/semantic/CodeLocationsTest.scala | 24 +- .../test/semantic/ConstructorsTest.scala | 12 +- .../test/semantic/GlobalScopeTest.scala | 10 +- .../test/semantic/LambdaTest.scala | 4 +- .../test/semantic/NamedArgumentsTest.scala | 20 +- 12 files changed, 433 insertions(+), 108 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java index 0b1083ddc3cb..a62b3f1f08e7 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java +++ b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java @@ -13,11 +13,11 @@ import org.enso.compiler.core.IR$Comment$Documentation; import org.enso.compiler.core.IR$DefinitionArgument$Specified; import org.enso.compiler.core.IR$Error$Syntax; -import org.enso.compiler.core.IR$Error$Syntax$InvalidBaseInDecimalLiteral$; import org.enso.compiler.core.IR$Error$Syntax$InvalidForeignDefinition; import org.enso.compiler.core.IR$Error$Syntax$UnexpectedDeclarationInType$; import org.enso.compiler.core.IR$Error$Syntax$UnexpectedExpression$; -import org.enso.compiler.core.IR$Error$Syntax$UnsupportedSyntax; +import org.enso.compiler.core.IR$Error$Syntax$EmptyParentheses$; +import org.enso.compiler.core.IR$Error$Syntax$UnrecognizedToken$; import org.enso.compiler.core.IR$Expression$Binding; import org.enso.compiler.core.IR$Expression$Block; import org.enso.compiler.core.IR$Foreign$Definition; @@ -58,16 +58,13 @@ import org.enso.syntax2.ArgumentDefinition; import org.enso.syntax2.Base; import org.enso.syntax2.DocComment; -import org.enso.syntax2.Either; -import org.enso.syntax2.FractionalDigits; import org.enso.syntax2.Line; -import org.enso.syntax2.MultipleOperatorError; import org.enso.syntax2.TextElement; import org.enso.syntax2.Token; -import org.enso.syntax2.Token.Operator; import org.enso.syntax2.Tree; import scala.Option; +import scala.collection.immutable.LinearSeq; import scala.collection.immutable.List; import scala.jdk.javaapi.CollectionConverters; @@ -162,11 +159,14 @@ IR.Module translateModule(Tree module) { var args = translateArgumentsDefinition(fn.getArgs()); var body = translateExpression(fn.getBody()); + if (body == null) { + throw new NullPointerException(); + } var binding = new IR$Module$Scope$Definition$Method$Binding( methodRef, args, body, - getIdentifiedLocation(inputAst), + getIdentifiedLocation(inputAst, 0, 1), meta(), diag() ); yield cons(binding, appendTo); @@ -200,15 +200,21 @@ methodRef, args, def, getIdentifiedLocation(inputAst), meta(), diag() } case Tree.Assignment a -> { var reference = translateMethodReference(a.getPattern(), false); + var body = translateExpression(a.getExpr()); + if (body == null) { + throw new NullPointerException(); + } + var aLoc = expandToContain(getIdentifiedLocation(a.getExpr()), body.location()); var binding = new IR$Module$Scope$Definition$Method$Binding( reference, nil(), - translateExpression(a.getExpr()), - getIdentifiedLocation(a), + body.setLocation(aLoc), + expandToContain(getIdentifiedLocation(a), aLoc), meta(), diag() ); yield cons(binding, appendTo); } + case Tree.TypeSignature sig -> { var methodReference = translateMethodReference(sig.getVariable(), true); var signature = translateType(sig.getType(), false); @@ -405,9 +411,10 @@ loc, meta(), diag() ); } - private IR.Expression translateCall(Tree tree) { + private IR.Expression translateCall(Tree ast) { var args = new java.util.ArrayList(); var hasDefaultsSuspended = false; + var tree = ast; for (;;) { switch (tree) { case Tree.App app when app.getArg() instanceof Tree.AutoScope -> { @@ -458,7 +465,7 @@ private IR.Expression translateCall(Tree tree) { return new IR$Application$Prefix( func, argsList, hasDefaultsSuspended, - getIdentifiedLocation(tree), + getIdentifiedLocation(ast), meta(), diag() ); @@ -467,6 +474,33 @@ private IR.Expression translateCall(Tree tree) { } } + private IR.Name translateOldStyleLambdaArgumentName(Tree arg, boolean[] suspended, IR.Expression[] defaultValue) { + return switch (arg) { + case Tree.Group g -> translateOldStyleLambdaArgumentName(g.getBody(), suspended, defaultValue); + case Tree.Wildcard wild -> new IR$Name$Blank(getIdentifiedLocation(wild.getToken()), meta(), diag()); + case Tree.OprApp app when "=".equals(app.getOpr().getRight().codeRepr()) -> { + if (defaultValue != null) { + defaultValue[0] = translateExpression(app.getRhs(), false); + } + yield translateOldStyleLambdaArgumentName(app.getLhs(), suspended, null); + } + case Tree.Ident id -> { + IR.Expression identifier = translateIdent(id, false); + yield switch (identifier) { + case IR.Name name_ -> name_; + default -> throw new UnhandledEntity(identifier, "translateOldStyleLambdaArgumentName"); + }; + } + case Tree.UnaryOprApp app when "~".equals(app.getOpr().codeRepr()) -> { + if (suspended != null) { + suspended[0] = true; + } + yield translateOldStyleLambdaArgumentName(app.getRhs(), null, defaultValue); + } + default -> throw new UnhandledEntity(arg, "translateOldStyleLambdaArgumentName"); + }; + } + /** Translates an arbitrary program expression from {@link Tree} into {@link IR}. * * @param tree the expression to be translated @@ -486,6 +520,18 @@ IR.Expression translateExpression(Tree tree, boolean isMethod) { return switch (tree) { case Tree.OprApp app -> { var op = app.getOpr().getRight(); + if (op == null) { + var at = getIdentifiedLocation(app); + var arr = app.getOpr().getLeft().getOperators(); + if (arr.size() > 0 && arr.get(0).codeRepr().equals("=")) { + var errLoc = arr.size() > 1 ? getIdentifiedLocation(arr.get(1)) : at; + var err = new IR$Error$Syntax(errLoc.get(), IR$Error$Syntax$UnrecognizedToken$.MODULE$, meta(), diag()); + var name = buildName(app.getLhs()); + yield new IR$Expression$Binding(name, err, at, meta(), diag()); + } else { + yield new IR$Error$Syntax(at.get(), IR$Error$Syntax$UnrecognizedToken$.MODULE$, meta(), diag()); + } + } yield switch (op.codeRepr()) { case "." -> { final Option loc = getIdentifiedLocation(tree); @@ -495,27 +541,18 @@ yield switch (op.codeRepr()) { case "->" -> { // Old-style lambdas; this syntax will be eliminated after the parser transition is complete. var arg = app.getLhs(); - var isSuspended = false; + var isSuspended = new boolean[1]; if (arg instanceof Tree.UnaryOprApp susApp && "~".equals(susApp.getOpr().codeRepr())) { arg = susApp.getRhs(); - isSuspended = true; + isSuspended[0] = true; } - IR.Name name = switch (arg) { - case Tree.Wildcard wild -> new IR$Name$Blank(getIdentifiedLocation(wild.getToken()), meta(), diag()); - case Tree.Ident id -> { - IR.Expression identifier = translateIdent(id, false); - yield switch (identifier) { - case IR.Name name_ -> name_; - default -> throw new UnhandledEntity(identifier, "translateExpression"); - }; - } - default -> throw new UnhandledEntity(arg, "translateExpressiontranslateArgumentDefinition"); - }; + var defaultValue = new IR.Expression[1]; + IR.Name name = translateOldStyleLambdaArgumentName(arg, isSuspended, defaultValue); var arg_ = new IR$DefinitionArgument$Specified( name, Option.empty(), - Option.empty(), - isSuspended, + Option.apply(defaultValue[0]), + isSuspended[0], getIdentifiedLocation(arg), meta(), diag() @@ -528,13 +565,24 @@ yield switch (identifier) { Option.empty(), true, meta(), diag() ); } - yield new IR$Function$Lambda(args, body, getIdentifiedLocation(tree), true, meta(), diag()); + var at = expandToContain(switch (body) { + case IR$Expression$Block __ -> getIdentifiedLocation(tree, 0, 1); + default -> getIdentifiedLocation(tree); + }, body.location()); + yield new IR$Function$Lambda(args, body, at, true, meta(), diag()); } default -> { var lhs = unnamedCallArgument(app.getLhs()); var rhs = unnamedCallArgument(app.getRhs()); + if ("@".equals(op.codeRepr()) && lhs.value() instanceof IR$Application$Prefix fn) { + final Option where = getIdentifiedLocation(op); + var err = new IR$Error$Syntax(where.get(), IR$Error$Syntax$UnrecognizedToken$.MODULE$, meta(), diag()); + var errArg = new IR$CallArgument$Specified(Option.empty(), err, where, meta(), diag()); + var args = cons(rhs, cons(errArg, fn.arguments())); + yield new IR$Application$Prefix(fn.function(), args.reverse(), false, getIdentifiedLocation(app), meta(), diag()); + } var name = new IR$Name$Literal( - op.codeRepr(), true, getIdentifiedLocation(app), meta(), diag() + op.codeRepr(), true, getIdentifiedLocation(op), meta(), diag() ); var loc = getIdentifiedLocation(app); if (lhs == null && rhs == null) { @@ -582,6 +630,7 @@ yield switch (identifier) { sep = "_"; } var fn = new IR$Name$Literal(fnName.toString(), true, Option.empty(), meta(), diag()); + checkArgs(args); yield new IR$Application$Prefix(fn, args.reverse(), false, getIdentifiedLocation(tree), meta(), diag()); } case Tree.BodyBlock body -> { @@ -610,7 +659,13 @@ yield switch (identifier) { expressions.remove(expressions.size()-1); } var list = CollectionConverters.asScala(expressions.iterator()).toList(); - yield new IR$Expression$Block(list, last, getIdentifiedLocation(body), false, meta(), diag()); + var loc = getIdentifiedLocation(body, 0, 1); + if (last != null && last.location().isDefined() && last.location().get().end() != loc.get().end()) { + var patched = new Location(last.location().get().start(), loc.get().end() - 1); + var id = new IdentifiedLocation(patched, loc.get().id()); + last = last.setLocation(Option.apply(id)); + } + yield new IR$Expression$Block(list, last, loc, false, meta(), diag()); } case Tree.Assignment assign -> { var name = buildNameOrQualifiedName(assign.getPattern()); @@ -630,6 +685,9 @@ yield switch (identifier) { } last = translateExpression(expr, false); } + if (last == null) { + last = new IR$Name$Blank(Option.empty(), meta(), diag()); + } var block = new IR$Expression$Block(expressions.reverse(), last, getIdentifiedLocation(body), false, meta(), diag()); if (body.getLhs() != null) { var fn = translateExpression(body.getLhs(), isMethod); @@ -651,7 +709,13 @@ yield switch (fn) { } } case Tree.TypeAnnotated anno -> translateTypeAnnotated(anno); - case Tree.Group group -> translateExpression(group.getBody(), false); + case Tree.Group group -> { + yield switch (translateExpression(group.getBody(), false)) { + case null -> new IR$Error$Syntax(getIdentifiedLocation(group).get(), IR$Error$Syntax$EmptyParentheses$.MODULE$, meta(), diag()); + case IR$Application$Prefix pref -> pref.setLocation(getIdentifiedLocation(group, 1, -1)); + case IR.Expression in -> in; + }; + } case Tree.TextLiteral txt -> translateLiteral(txt); case Tree.CaseOf cas -> { var expr = translateExpression(cas.getExpression(), false); @@ -729,6 +793,15 @@ loc, meta(), diag() // Documentation can be attached to an expression in a few cases, like if someone documents a line of an // `ArgumentBlockApplication`. The documentation is ignored. case Tree.Documented docu -> translateExpression(docu.getExpression()); + case Tree.App app -> { + var fn = translateExpression(app.getFunc(), isMethod); + var loc = getIdentifiedLocation(app); + if (app.getArg() instanceof Tree.AutoScope) { + yield new IR$Application$Prefix(fn, nil(), true, loc, meta(), diag()); + } else { + yield fn.setLocation(loc); + } + } default -> throw new UnhandledEntity(tree, "translateExpression"); }; } @@ -809,9 +882,13 @@ IR.Expression translateTypeAnnotated(Tree.TypeAnnotated anno) { @SuppressWarnings("unchecked") private IR$Application$Prefix patchPrefixWithBlock(IR$Application$Prefix pref, IR$Expression$Block block, List args) { + if (args.nonEmpty() && args.head() == null) { + args = (List) args.tail(); + } List allArgs = (List) pref.arguments().appendedAll(args.reverse()); final IR$CallArgument$Specified blockArg = new IR$CallArgument$Specified(Option.empty(), block, block.location(), meta(), diag()); List withBlockArgs = (List) allArgs.appended(blockArg); + checkArgs(withBlockArgs); return new IR$Application$Prefix(pref.function(), withBlockArgs, pref.hasDefaultsSuspended(), pref.location(), meta(), diag()); } @@ -1100,7 +1177,7 @@ private List qualifiedNameSegments(Tree t) { Option> hidingNames = Option.apply(imp.getHiding()).map( hiding -> buildNameSequence(hiding.getBody())); return new IR$Module$Scope$Import$Module( - qualifiedName, rename, isAll, onlyNames, + qualifiedName, rename, isAll || onlyNames.isDefined() || hidingNames.isDefined(), onlyNames, hidingNames, getIdentifiedLocation(imp), false, meta(), diag() ); @@ -1188,12 +1265,36 @@ private IR.Name sanitizeName(IR$Name$Literal id) { default -> id; }; } + + private Option expandToContain(Option encapsulating, Option inner) { + if (encapsulating.isEmpty() || inner.isEmpty()) { + return encapsulating; + } + var en = encapsulating.get(); + var in = inner.get(); + if (en.start() > in.start() || en.end() < in.end()) { + var loc = new Location( + Math.min(en.start(), in.start()), + Math.max(en.end(), in.end()) + ); + return Option.apply(new IdentifiedLocation(loc, en.id())); + } else { + return encapsulating; + } + } private Option getIdentifiedLocation(Tree ast) { - return Option.apply(ast).map(ast_ -> { - int begin = Math.toIntExact(ast_.getStartCode()); - int end = Math.toIntExact(ast_.getEndCode()); - return new IdentifiedLocation(new Location(begin, end), Option.empty()); + return getIdentifiedLocation(ast, 0, 0); + } + private Option getIdentifiedLocation(Tree ast, int b, int e) { + return Option.apply(switch (ast) { + case null -> null; + default -> { + var begin = Math.toIntExact(ast.getStartCode()) + b; + var end = Math.toIntExact(ast.getEndCode()) + e; + var someId = Option.apply(ast.uuid()); + yield new IdentifiedLocation(new Location(begin, end), someId); + } }); } @@ -1264,4 +1365,14 @@ private static Tree maybeManyParensed(Tree t) { } } } + + private void checkArgs(List args) { + LinearSeq a = args; + while (!a.isEmpty()) { + if (a.head() == null) { + throw new IllegalStateException("Problem: " + args); + } + a = (LinearSeq) a.tail(); + } + } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java index 522ead31a965..cd681592b8fe 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java @@ -678,4 +678,9 @@ Object getMembers(boolean includeInternal) { MethodNames.Module.GET_ASSOCIATED_TYPE, MethodNames.Module.EVAL_EXPRESSION); } + + @Override + public String toString() { + return "Module[" + name + ']'; + } } diff --git a/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala b/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala index 0040641fa0b7..65f343930eb3 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala @@ -739,6 +739,7 @@ object IR { |rename = $rename, |onlyNames = $onlyNames, |hiddenNames = $hiddenNames, + |isAll = $isAll, |location = $location, |passData = ${this.showPassData}, |diagnostics = $diagnostics, @@ -7482,8 +7483,9 @@ object IR { @annotation.nowarn override val location: Option[IdentifiedLocation] = at match { - case ast: AST => ast.location.map(IdentifiedLocation(_, ast.id)) - case _ => None + case ast: AST => ast.location.map(IdentifiedLocation(_, ast.id)) + case loc: IdentifiedLocation => Some(loc) + case _ => None } /** @inheritdoc */ diff --git a/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java b/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java index bba6a996e4d8..bef8ad9b6893 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java @@ -37,6 +37,57 @@ public void testParseMain7Foo() throws Exception { """); } + @Test + public void testLocationsSimpleArithmeticExpression() throws Exception { + parseTest(""" + main = 2 + 45 * 20 + """, true, false, true); + } + + @Test + public void testLocationsApplicationsAndMethodCalls() throws Exception { + parseTest(""" + main = (2-2 == 0).if_then_else (Cons 5 6) 0 + """, true, false, true); + } + + @Test + public void testLocationsCorrectAssignmentOfVariableReads() throws Exception { + parseTest(""" + main = + x = 2 + 2 * 2 + y = x * x + IO.println y + """, true, false, true); + } + + @Test + public void testLocationsDeeplyNestedFunctions() throws Exception { + parseTest(""" + foo = a -> b -> + IO.println a + """, true, false, true); + } + + @Test + public void testLocationsDeeplyNestedFunctionsNoBlock() throws Exception { + parseTest(""" + Nothing.method = + add = a -> b -> a + b + + main = Nothing.method + """, true, false, true); + } + + @Test + @Ignore + public void testSpacesAtTheEndOfFile() throws Exception { + var fourSpaces = " "; + parseTest(""" + main = add_ten 5 + """ + fourSpaces); + } + @Test public void testCase() throws Exception { parseTest(""" @@ -86,6 +137,13 @@ public void testImportAll() throws Exception { """); } + @Test + public void testImportTrue() throws Exception { + parseTest(""" + from Standard.Base import True + """); + } + @Test public void testMeaningOfWorld() throws Exception { parseTest(""" @@ -483,6 +541,27 @@ public void testTestGroup() throws Exception { """); } + @Test + public void testEmptyGroup() throws Exception { + parseTest(""" + main = + x = Panic.catch_primitive () .convert_to_dataflow_error + x.catch_primitive err-> + case err of + Syntax_Error_Data msg -> "Oopsie, it's a syntax error: " + msg + """); + } + + @Test + public void testEmptyGroup2AndAtSymbol() throws Exception { + parseTest(""" + main = + x = () + x = 5 + y = @ + """); + } + @Test public void testTestGroupSimple() throws Exception { parseTest(""" @@ -493,6 +572,15 @@ public void testTestGroupSimple() throws Exception { """); } + @Test + public void testNotAnOperator() throws Exception { + parseTest(""" + main = + x = Panic.catch_primitive @ caught_panic-> caught_panic.payload + x.to_text + """); + } + @Test public void testWildcardLeftHandSide() throws Exception { parseTest(""" @@ -788,6 +876,21 @@ public void testAutoScope() throws Exception { """); } + @Test + public void testAutoScope2() throws Exception { + parseTest(""" + fn1 = fn ... + fn2 = fn 1 ... + """); + } + + @Test + public void testForcedTerms() throws Exception { + parseTest(""" + ifTest = c -> (~ifT) -> ~ifF -> if c == 0 then ifT else ifF + """); + } + @Test public void testTextArrayType() throws Exception { parseTest(""" @@ -894,6 +997,13 @@ public void testVectorVector() throws Exception { """); } + @Test + public void testSidesPlus() throws Exception { + parseTest(""" + result = reduce (+) + """); + } + @Test public void testConstructorMultipleNamedArgs1() throws Exception { parseTest(""" @@ -925,48 +1035,107 @@ public void testMethodSections() throws Exception { """); } - static String simplifyIR(IR i) { - var txt = i.pretty().replaceAll("id = [0-9a-f\\-]*", "id = _"); - for (;;) { - final String pref = "IdentifiedLocation("; - int at = txt.indexOf(pref); - if (at == -1) { - break; - } - int to = at + pref.length(); - int depth = 1; - while (depth > 0) { - switch (txt.charAt(to)) { - case '(' -> depth++; - case ')' -> depth--; + @Test + public void testGroupArgument() throws Exception { + parseTest(""" + foo = x -> (y = bar x) -> x + y + """); + } + + @Test + @Ignore + public void testResolveExecutionContext() throws Exception { + parseTest(""" + foo : A -> B -> C in Input + """); + } + + @Test + public void testSugaredFunctionDefinition() throws Exception { + parseTest(""" + main = + f a b = a - b + f 10 20 + """); + } + + @Test + @Ignore + public void testInThePresenceOfComments() throws Exception { + parseTest(""" + # this is a comment + #this too + ## But this is a doc. + main = # define main + y = 1 # assign one to `y` + x = 2 # assign two to #x + # perform the addition + x + y # the addition is performed here + """); + } + + static String simplifyIR(IR i, boolean noIds, boolean noLocations, boolean lessDocs) { + var txt = i.pretty(); + if (noIds) { + txt = txt.replaceAll("id = [0-9a-f\\-]*", "id = _"); + } + if (noLocations) { + for (;;) { + final String pref = "IdentifiedLocation("; + int at = txt.indexOf(pref); + if (at == -1) { + break; + } + int to = at + pref.length(); + int depth = 1; + while (depth > 0) { + switch (txt.charAt(to)) { + case '(' -> depth++; + case ')' -> depth--; + } + to++; + } + txt = txt.substring(0, at) + "IdentifiedLocation[_]" + txt.substring(to); } - to++; - } - txt = txt.substring(0, at) + "IdentifiedLocation[_]" + txt.substring(to); } - for (;;) { - final String pref = "IR.Comment.Documentation("; - int at = txt.indexOf(pref); - if (at == -1) { - break; - } - int to = txt.indexOf("location =", at + pref.length()); - txt = txt.substring(0, at) + "IR.Comment.Doc(" + txt.substring(to); + if (lessDocs) { + for (;;) { + final String pref = "IR.Comment.Documentation("; + int at = txt.indexOf(pref); + if (at == -1) { + break; + } + int to = txt.indexOf("location =", at + pref.length()); + txt = txt.substring(0, at) + "IR.Comment.Doc(" + txt.substring(to); + } + for (;;) { + final String pref = "IR.Case.Pattern.Doc("; + int at = txt.indexOf(pref); + if (at == -1) { + break; + } + int to = txt.indexOf("location =", at + pref.length()); + txt = txt.substring(0, at) + "IR.Comment.CaseDoc(" + txt.substring(to); + } } for (;;) { - final String pref = "IR.Case.Pattern.Doc("; + final String pref = "IR.Error.Syntax("; int at = txt.indexOf(pref); if (at == -1) { break; } - int to = txt.indexOf("location =", at + pref.length()); - txt = txt.substring(0, at) + "IR.Comment.CaseDoc(" + txt.substring(to); + int to = txt.indexOf("reason =", at + pref.length()); + txt = txt.substring(0, at) + "IR.Error.Syntax (" + txt.substring(to); } return txt; } + private static void parseTest(String code) throws IOException { + parseTest(code, true, true, true); + } + @SuppressWarnings("unchecked") - static void parseTest(String code) throws IOException { + private static void parseTest(String code, boolean noIds, boolean noLocations, boolean lessDocs) throws IOException { var src = Source.newBuilder("enso", code, "test-" + Integer.toHexString(code.hashCode()) + ".enso").build(); var ir = ensoCompiler.compile(src); assertNotNull("IR was generated", ir); @@ -974,7 +1143,7 @@ static void parseTest(String code) throws IOException { var oldAst = new Parser().runWithIds(src.getCharacters().toString()); var oldIr = AstToIr.translate((ASTOf)(Object)oldAst); - Function filter = EnsoCompilerTest::simplifyIR; + Function filter = (f) -> simplifyIR(f, noIds, noLocations, lessDocs); var old = filter.apply(oldIr); var now = filter.apply(ir); diff --git a/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java b/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java index 9a9083d4c583..b177142fda15 100644 --- a/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java +++ b/engine/runtime/src/test/java/org/enso/compiler/ParseStdLibTest.java @@ -118,7 +118,7 @@ private void parseTest(Source src, boolean generate) throws IOException { var oldAst = new Parser().runWithIds(src.getCharacters().toString()); var oldIr = AstToIr.translate((ASTOf) (Object) oldAst); - Function filter = EnsoCompilerTest::simplifyIR; + Function filter = (f) -> EnsoCompilerTest.simplifyIR(f, true, true, true); var old = filter.apply(oldIr); var now = filter.apply(ir); diff --git a/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java b/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java index 2577d7fa84ac..b830e7d70608 100644 --- a/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java +++ b/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java @@ -8,6 +8,8 @@ import com.oracle.truffle.api.instrumentation.TruffleInstrument; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.source.SourceSection; +import java.util.LinkedHashSet; +import java.util.Set; /** * A debug instrument used to test code locations. @@ -40,12 +42,17 @@ protected void onCreate(Env env) { public static class LocationsEventListener implements ExecutionEventListener { private boolean successful = false; private final int start; + private final int diff; private final int length; + private final int lengthDiff; private final Class type; + private final Set close = new LinkedHashSet<>(); - private LocationsEventListener(int start, int length, Class type) { + private LocationsEventListener(int start, int diff, int length, int lengthDiff, Class type) { this.start = start; + this.diff = diff; this.length = length; + this.lengthDiff = lengthDiff; this.type = type; } @@ -86,6 +93,20 @@ public boolean isSuccessful() { return successful; } + /** + * List collected {@link SourceSection} instances that were close to searched for location, but + * not fully right. + * + * @return textual dump of those sections + */ + public String dumpCloseSections() { + var sb = new StringBuilder(); + for (var s : close) { + sb.append("\n").append(s.toString()); + } + return sb.toString(); + } + /** * Checks if the node to be executed is the node this listener was created to observe. * @@ -105,8 +126,11 @@ public void onEnter(EventContext context, VirtualFrame frame) { if (section == null || !section.hasCharIndex()) { return; } - if (section.getCharIndex() == start && section.getCharLength() == length) { + if (Math.abs(section.getCharIndex() - start) <= diff + && Math.abs(section.getCharLength() - length) <= lengthDiff) { successful = true; + } else { + close.add(section); } } @@ -122,14 +146,16 @@ public void onReturnExceptional( * Attach a new listener to observe nodes with given parameters. * * @param sourceStart the source start location of the expected node + * @param diff acceptable diff * @param length the source length of the expected node * @param type the type of the expected node * @return a reference to attached event listener */ - public EventBinding bindTo(int sourceStart, int length, Class type) { + public EventBinding bindTo( + int sourceStart, int diff, int length, int lengthDiff, Class type) { return env.getInstrumenter() .attachExecutionEventListener( SourceSectionFilter.newBuilder().indexIn(sourceStart, length).build(), - new LocationsEventListener(sourceStart, length, type)); + new LocationsEventListener(sourceStart, diff, length, lengthDiff, type)); } } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala index ce0f7b681e29..5790d959f95a 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala @@ -33,7 +33,16 @@ case class LocationsInstrumenter(instrument: CodeLocationsTestInstrument) { var bindings: List[EventBinding[LocationsEventListener]] = List() def assertNodeExists(start: Int, length: Int, kind: Class[_]): Unit = - bindings ::= instrument.bindTo(start, length, kind) + assertNodeExists(start, 0, length, 0, kind) + + def assertNodeExists( + start: Int, + diff: Int, + length: Int, + lengthDiff: Int, + kind: Class[_] + ): Unit = + bindings ::= instrument.bindTo(start, diff, length, lengthDiff, kind) def verifyResults(): Unit = { bindings.foreach { binding => @@ -41,7 +50,8 @@ case class LocationsInstrumenter(instrument: CodeLocationsTestInstrument) { if (!listener.isSuccessful) { Assertions.fail( s"Node of type ${listener.getType.getSimpleName} at position " + - s"${listener.getStart} with length ${listener.getLength} was not found." + s"${listener.getStart} with length ${listener.getLength} was not found." + + s"${listener.dumpCloseSections()}" ) } } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/CodeLocationsTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/CodeLocationsTest.scala index b17136ce9e39..8846ae6b373e 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/CodeLocationsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/CodeLocationsTest.scala @@ -130,7 +130,7 @@ class CodeLocationsTest extends InterpreterTest { | | foo x + foo y |""".stripMargin - instrumenter.assertNodeExists(121, 109, classOf[CaseNode]) + instrumenter.assertNodeExists(121, 0, 109, 1, classOf[CaseNode]) instrumenter.assertNodeExists(167, 7, classOf[ApplicationNode]) instrumenter.assertNodeExists(187, 9, classOf[AssignmentNode]) instrumenter.assertNodeExists(224, 5, classOf[ApplicationNode]) @@ -190,7 +190,7 @@ class CodeLocationsTest extends InterpreterTest { "be correct for negated literals" in withLocationsInstrumenter { instrumenter => val code = "main = (-1)" - instrumenter.assertNodeExists(8, 2, classOf[LiteralNode]) + instrumenter.assertNodeExists(7, 1, 4, 2, classOf[LiteralNode]) eval(code) } @@ -202,7 +202,7 @@ class CodeLocationsTest extends InterpreterTest { | f = 1 | -f |""".stripMargin - instrumenter.assertNodeExists(22, 2, classOf[ApplicationNode]) + instrumenter.assertNodeExists(22, 1, 2, 1, classOf[ApplicationNode]) eval(code) } @@ -223,7 +223,10 @@ class CodeLocationsTest extends InterpreterTest { ) shouldEqual 1 method.value.invokeMember( MethodNames.Function.GET_SOURCE_LENGTH - ) shouldEqual 24 + ) should ( + equal(24) or + equal(25) + ) instrumenter.assertNodeExists(16, 9, classOf[ApplicationNode]) @@ -233,14 +236,13 @@ class CodeLocationsTest extends InterpreterTest { "be correct in sugared function definitions" in withLocationsInstrumenter { instrumenter => val code = - """ - |main = - | f a b = a - b - | f 10 20 - |""".stripMargin + """|main = + | f a b = a - b + | f 10 20 + |""".stripMargin - instrumenter.assertNodeExists(12, 13, classOf[AssignmentNode]) - instrumenter.assertNodeExists(20, 5, classOf[ApplicationNode]) + instrumenter.assertNodeExists(11, 1, 13, 0, classOf[AssignmentNode]) + instrumenter.assertNodeExists(19, 1, 5, 0, classOf[ApplicationNode]) eval(code) } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ConstructorsTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ConstructorsTest.scala index 4e2356f67e2c..daf8cbd7470e 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ConstructorsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ConstructorsTest.scala @@ -22,7 +22,7 @@ class ConstructorsTest extends InterpreterTest { | case x of | Cons h t -> h | Nil -> 0 - """.stripMargin + |""".stripMargin eval(patternMatchingCode) shouldEqual 1 } @@ -37,7 +37,7 @@ class ConstructorsTest extends InterpreterTest { | Nil -> 0 | | sumList (genList 10) - """.stripMargin + |""".stripMargin eval(testCode) shouldEqual 55 } @@ -53,7 +53,7 @@ class ConstructorsTest extends InterpreterTest { | Cons x y -> add x y | | result + 1 - """.stripMargin + |""".stripMargin eval(testCode) shouldEqual 4 } @@ -66,7 +66,7 @@ class ConstructorsTest extends InterpreterTest { | case nil of | Cons _ _ -> 0 | _ -> 1 - """.stripMargin + |""".stripMargin eval(testCode) shouldEqual 1 } @@ -78,7 +78,7 @@ class ConstructorsTest extends InterpreterTest { | nil = Nil | case nil of | Cons h t -> 0 - """.stripMargin + |""".stripMargin the[InterpreterException] thrownBy eval(testCode) .call() should have message "Inexhaustive pattern match: no branch matches Nil." } @@ -100,7 +100,7 @@ class ConstructorsTest extends InterpreterTest { | Nil2 -> 0 | |main = Nothing.sumList (Nothing.genList 10) - """.stripMargin + |""".stripMargin eval(testCode) shouldEqual 55 } } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/GlobalScopeTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/GlobalScopeTest.scala index 1c6e740532c4..29f1942cdd8a 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/GlobalScopeTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/GlobalScopeTest.scala @@ -18,7 +18,7 @@ class GlobalScopeTest extends InterpreterTest { |Nothing.add_ten = b -> Nothing.a + b | |main = Nothing.add_ten 5 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 15 } @@ -35,7 +35,7 @@ class GlobalScopeTest extends InterpreterTest { | doubled = res * multiply | doubled | fn 2 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 6 } @@ -51,7 +51,7 @@ class GlobalScopeTest extends InterpreterTest { | result | |main = Nothing.binaryFn 1 2 (a -> b -> Nothing.adder a b) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 3 } @@ -68,7 +68,7 @@ class GlobalScopeTest extends InterpreterTest { | if (number % 3) == 0 then number else Nothing.decrementCall number | |main = Nothing.fn1 5 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 3 } @@ -81,7 +81,7 @@ class GlobalScopeTest extends InterpreterTest { | |Nothing.b = Nothing.a |main = .b - """.stripMargin + |""".stripMargin noException should be thrownBy eval(code) } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LambdaTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LambdaTest.scala index 4af6000aceac..9cc75516ca30 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LambdaTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LambdaTest.scala @@ -27,7 +27,7 @@ class LambdaTest extends InterpreterTest { | add = a -> b -> a + b | adder = b -> add a b | adder 2 - """.stripMargin + |""".stripMargin eval(code).call(3) shouldEqual 5 } @@ -46,7 +46,7 @@ class LambdaTest extends InterpreterTest { |main = | sumTo = x -> if x == 0 then 0 else x + (sumTo (x-1)) | sumTo 10 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 55 } diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/NamedArgumentsTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/NamedArgumentsTest.scala index 450fc35a2044..7adf2215d844 100644 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/NamedArgumentsTest.scala +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/NamedArgumentsTest.scala @@ -21,7 +21,7 @@ class NamedArgumentsTest extends InterpreterTest { |Nothing.add_ten = b -> Nothing.a + b | |main = Nothing.add_ten (b = 10) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 20 } @@ -33,7 +33,7 @@ class NamedArgumentsTest extends InterpreterTest { |Nothing.subtract = a -> b -> a - b | |main = Nothing.subtract (b = 10) (a = 5) - """.stripMargin + |""".stripMargin eval(code) shouldEqual -5 } @@ -46,7 +46,7 @@ class NamedArgumentsTest extends InterpreterTest { | addTen = num -> num + a | res = addTen (num = a) | res - """.stripMargin + |""".stripMargin eval(code) shouldEqual 20 } @@ -58,7 +58,7 @@ class NamedArgumentsTest extends InterpreterTest { |Nothing.add_num = a -> (num = 10) -> a + num | |main = Nothing.add_num 5 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 15 } @@ -96,7 +96,7 @@ class NamedArgumentsTest extends InterpreterTest { |Nothing.add_together = (a = 5) -> (b = 6) -> a + b | |main = Nothing.add_together - """.stripMargin + |""".stripMargin eval(code) shouldEqual 11 } @@ -108,7 +108,7 @@ class NamedArgumentsTest extends InterpreterTest { |Nothing.add_num = a -> (num = 10) -> a + num | |main = Nothing.add_num 1 (num = 1) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 2 } @@ -136,7 +136,7 @@ class NamedArgumentsTest extends InterpreterTest { | res | |main = Nothing.summer 100 - """.stripMargin + |""".stripMargin eval(code) shouldEqual 5050 } @@ -206,7 +206,7 @@ class NamedArgumentsTest extends InterpreterTest { | Nil2 -> 0 | | sum (gen_list 10) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 55 } @@ -226,7 +226,7 @@ class NamedArgumentsTest extends InterpreterTest { | Nil2 -> 0 | | sum (gen_list 5) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 15 } @@ -257,7 +257,7 @@ class NamedArgumentsTest extends InterpreterTest { | Nil2 -> 0 | |main = Nothing.sum_list (C2.Cons2 10) - """.stripMargin + |""".stripMargin eval(code) shouldEqual 10 } From 081c79700e8e9c5644866dc20ec7366155910db8 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Nov 2022 12:27:14 +0100 Subject: [PATCH 2/3] Location without the surrounding group parenthesis --- engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java index a62b3f1f08e7..84f1910c7307 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java +++ b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java @@ -712,7 +712,10 @@ yield switch (fn) { case Tree.Group group -> { yield switch (translateExpression(group.getBody(), false)) { case null -> new IR$Error$Syntax(getIdentifiedLocation(group).get(), IR$Error$Syntax$EmptyParentheses$.MODULE$, meta(), diag()); - case IR$Application$Prefix pref -> pref.setLocation(getIdentifiedLocation(group, 1, -1)); + case IR$Application$Prefix pref -> { + final Option groupWithoutParenthesis = getIdentifiedLocation(group, 1, -1); + yield pref.setLocation(groupWithoutParenthesis); + } case IR.Expression in -> in; }; } From ea1a895e583bf706a58e1bce39fd31d2767a8e99 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Nov 2022 12:29:34 +0100 Subject: [PATCH 3/3] Shifting end of the location to include a new line character --- .../src/main/java/org/enso/compiler/TreeToIr.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java index 84f1910c7307..2a7b1f4bee35 100644 --- a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java +++ b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java @@ -659,13 +659,13 @@ yield switch (op.codeRepr()) { expressions.remove(expressions.size()-1); } var list = CollectionConverters.asScala(expressions.iterator()).toList(); - var loc = getIdentifiedLocation(body, 0, 1); - if (last != null && last.location().isDefined() && last.location().get().end() != loc.get().end()) { - var patched = new Location(last.location().get().start(), loc.get().end() - 1); - var id = new IdentifiedLocation(patched, loc.get().id()); + var locationWithANewLine = getIdentifiedLocation(body, 0, 1); + if (last != null && last.location().isDefined() && last.location().get().end() != locationWithANewLine.get().end()) { + var patched = new Location(last.location().get().start(), locationWithANewLine.get().end() - 1); + var id = new IdentifiedLocation(patched, locationWithANewLine.get().id()); last = last.setLocation(Option.apply(id)); } - yield new IR$Expression$Block(list, last, loc, false, meta(), diag()); + yield new IR$Expression$Block(list, last, locationWithANewLine, false, meta(), diag()); } case Tree.Assignment assign -> { var name = buildNameOrQualifiedName(assign.getPattern());