Port graph editor to new AST (#4113)

Use the Rust parser rather than the Scala parser to parse Enso code in the IDE.

Implements:
- https://www.pivotaltracker.com/story/show/182975925
- https://www.pivotaltracker.com/story/show/182988419
- https://www.pivotaltracker.com/story/show/182970096
- https://www.pivotaltracker.com/story/show/182973659
- https://www.pivotaltracker.com/story/show/182974161
- https://www.pivotaltracker.com/story/show/182974205

There is additional functionality needed before the transition is fully-completed, however I think it's time for this to see review and testing, so I've opened separate issues. In rough order of urgency (these issues are also linked from the corresponding disabled tests):
- #5573
- #5571
- #5572
- #5574

# Important Notes
The implementation is based partly on translation, and partly on new analysis. Method- and operator-related shapes are translated to the old `Ast` variants, so that all the analysis applied to them doesn't need to be ported at this time. Everything else (mostly "macros" in the old AST) is implemented with new analysis.

Cargo.toml

@@ -5,6 +5,7 @@ resolver = "2"
 # where plausible.
 members = [
   "app/gui",
+  "app/gui/language/parser",
   "app/gui/enso-profiler-enso-data",
   "build/cli",
   "build/macros",

app/gui/Cargo.toml

@@ -32,10 +32,11 @@ ensogl-hardcoded-theme = { path = "../../lib/rust/ensogl/app/theme/hardcoded" }
 ensogl-drop-manager = { path = "../../lib/rust/ensogl/component/drop-manager" }
 fuzzly = { path = "../../lib/rust/fuzzly" }
 ast = { path = "language/ast/impl" }
+parser = { path = "language/parser" }
+parser-scala = { path = "language/parser-scala" }
 ide-view = { path = "view" }
 engine-protocol = { path = "controller/engine-protocol" }
 json-rpc = { path = "../../lib/rust/json-rpc" }
-parser-scala = { path = "language/parser" }
 span-tree = { path = "language/span-tree" }
 bimap = { version = "0.4.0" }
 console_error_panic_hook = { workspace = true }

app/gui/controller/double-representation/Cargo.toml

@@ -9,8 +9,8 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 ast = { path = "../../language/ast/impl" }
+parser = { path = "../../language/parser" }
 engine-protocol = { path = "../engine-protocol" }
-parser-scala = { path = "../../language/parser" }
 enso-data-structures = { path = "../../../../lib/rust/data-structures" }
 enso-prelude = { path = "../../../../lib/rust/prelude" }
 enso-profiler = { path = "../../../../lib/rust/profiler" }

app/gui/controller/double-representation/src/alias_analysis.rs

@@ -240,13 +240,6 @@ impl AliasAnalyzer {
             self.process_assignment(&assignment);
         } else if let Some(lambda) = ast::macros::as_lambda(ast) {
             self.process_lambda(&lambda);
-        } else if let Ok(macro_match) = ast::known::Match::try_from(ast) {
-            // Macros (except for lambdas which were covered in the previous check) never introduce
-            // new scopes or different context. We skip the keywords ("if" in "if-then-else" is not
-            // an identifier) and process the matched subtrees as usual.
-            self.process_given_subtrees(macro_match.shape(), macro_match.iter_pat_match_subcrumbs())
-        } else if let Ok(ambiguous) = ast::known::Ambiguous::try_from(ast) {
-            self.process_given_subtrees(ambiguous.shape(), ambiguous.iter_pat_match_subcrumbs())
         } else if self.is_in_pattern() {
             // We are in the pattern (be it a lambda's or assignment's left side). Three options:
             // 1) This is a destructuring pattern match using infix syntax, like `head,tail`.
@@ -371,8 +364,6 @@ mod tests {
     use super::test_utils::*;
     use super::*;
 
-    wasm_bindgen_test_configure!(run_in_browser);
-
     /// Checks if actual observed sequence of located identifiers matches the expected one.
     /// Expected identifiers are described as code spans in the node's text representation.
     fn validate_identifiers(
@@ -386,7 +377,7 @@ mod tests {
     }
 
     /// Runs the test for the given test case description.
-    fn run_case(parser: &parser_scala::Parser, case: Case) {
+    fn run_case(parser: &parser::Parser, case: Case) {
         debug!("\n===========================================================================\n");
         debug!("Case: {}", case.code);
         let ast = parser.parse_line_ast(&case.code).unwrap();
@@ -397,15 +388,15 @@ mod tests {
     }
 
     /// Runs the test for the test case expressed using markdown notation. See `Case` for details.
-    fn run_markdown_case(parser: &parser_scala::Parser, marked_code: impl AsRef<str>) {
+    fn run_markdown_case(parser: &parser::Parser, marked_code: impl AsRef<str>) {
         debug!("Running test case for {}", marked_code.as_ref());
         let case = Case::from_markdown(marked_code.as_ref());
         run_case(parser, case)
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn test_alias_analysis() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let test_cases = [
             "»foo«",
             "«five» = 5",
@@ -433,21 +424,11 @@ mod tests {
             "»A« -> »b«",
             "a -> »A« -> a",
             "a -> a -> »A«",
-            "x»,«y -> »B«",
-            "x»,«y -> y",
-            "x »,« »Y« -> _",
             "(»foo«)",
             "(«foo») = (»bar«)",
             "if »A« then »B«",
             "if »a« then »b« else »c«",
-            "case »foo« of\n    »Number« a -> a\n    »Wildcard« -> »bar«\n    a»,«b -> a",
-            // === Macros Ambiguous ===
             "(»foo«",
-            "if »a«",
-            "case »a«",
-            // "->»a«", // TODO [mwu] restore (and implement) when parser is able to parse this
-            // "a ->",  // TODO [mwu] restore (and implement) when parser is able to parse this
-
             // === Definition ===
             "«foo» a b c = »foo« a »d«",
             "«foo» a b c = d -> a d",

app/gui/controller/double-representation/src/connection.rs

@@ -158,7 +158,7 @@ mod tests {
 
     use ast::crumbs;
     use ast::crumbs::InfixCrumb;
-    use parser_scala::Parser;
+    use parser::Parser;
 
     struct TestRun {
         graph:       GraphInfo,
@@ -182,7 +182,7 @@ mod tests {
         }
 
         fn from_main_def(code: impl Str) -> TestRun {
-            let parser = Parser::new_or_panic();
+            let parser = Parser::new();
             let module = parser.parse_module(code, default()).unwrap();
             let definition = DefinitionInfo::from_root_line(&module.lines[0]).unwrap();
             Self::from_definition(definition)
@@ -199,15 +199,15 @@ mod tests {
         }
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     pub fn connection_listing_test_plain() {
         use InfixCrumb::LeftOperand;
         use InfixCrumb::RightOperand;
 
         let code_block = r"
-d,e = p
+d = p
 a = d
-b = e
+b = d
 c = a + b
 fun a = a b
 f = fun 2";
@@ -221,35 +221,35 @@ f = fun 2";
         assert_eq!(&c.destination.crumbs, &crumbs![RightOperand, LeftOperand]);
 
         let c = &run.connections[1];
-        assert_eq!(run.endpoint_node_repr(&c.source), "b = e");
+        assert_eq!(run.endpoint_node_repr(&c.source), "b = d");
         assert_eq!(&c.source.crumbs, &crumbs![LeftOperand]);
         assert_eq!(run.endpoint_node_repr(&c.destination), "c = a + b");
         assert_eq!(&c.destination.crumbs, &crumbs![RightOperand, RightOperand]);
 
         let c = &run.connections[2];
-        assert_eq!(run.endpoint_node_repr(&c.source), "d,e = p");
-        assert_eq!(&c.source.crumbs, &crumbs![LeftOperand, LeftOperand]);
+        assert_eq!(run.endpoint_node_repr(&c.source), "d = p");
+        assert_eq!(&c.source.crumbs, &crumbs![LeftOperand]);
         assert_eq!(run.endpoint_node_repr(&c.destination), "a = d");
         assert_eq!(&c.destination.crumbs, &crumbs![RightOperand]);
 
         let c = &run.connections[3];
-        assert_eq!(run.endpoint_node_repr(&c.source), "d,e = p");
-        assert_eq!(&c.source.crumbs, &crumbs![LeftOperand, RightOperand]);
-        assert_eq!(run.endpoint_node_repr(&c.destination), "b = e");
+        assert_eq!(run.endpoint_node_repr(&c.source), "d = p");
+        assert_eq!(&c.source.crumbs, &crumbs![LeftOperand]);
+        assert_eq!(run.endpoint_node_repr(&c.destination), "b = d");
         assert_eq!(&c.destination.crumbs, &crumbs![RightOperand]);
 
         // Note that line `fun a = a b` des not introduce any connections, as it is a definition.
 
         assert_eq!(run.connections.len(), 4);
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     pub fn inline_definition() {
         let run = TestRun::from_main_def("main = a");
         assert!(run.connections.is_empty());
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     pub fn listing_dependent_nodes() {
         let code_block = "\
             f,g = p
@@ -259,7 +259,6 @@ f = fun 2";
             d = a + b
             e = b";
         let mut expected_dependent_nodes = HashMap::<&'static str, Vec<&'static str>>::new();
-        expected_dependent_nodes.insert("f,g = p", vec!["a = f", "b = g", "d = a + b", "e = b"]);
         expected_dependent_nodes.insert("a = f", vec!["d = a + b"]);
         expected_dependent_nodes.insert("b = g", vec!["d = a + b", "e = b"]);
         expected_dependent_nodes.insert("c = 2", vec![]);

app/gui/controller/double-representation/src/definition.rs

@@ -11,7 +11,7 @@ use ast::crumbs::InfixCrumb;
 use ast::crumbs::Located;
 use ast::known;
 use ast::opr;
-use parser_scala::Parser;
+use parser::Parser;
 use std::iter::FusedIterator;
 
 
@@ -284,9 +284,7 @@ impl DefinitionInfo {
         let elem = line.elem.ok_or(MissingLineWithAst)?;
         let off = line.off;
         let first_line = ast::BlockLine { elem, off };
-        let is_orphan = false;
-        let ty = ast::BlockType::Discontinuous {};
-        let block = ast::Block { ty, indent, empty_lines, first_line, lines, is_orphan };
+        let block = ast::Block { indent, empty_lines, first_line, lines };
         let body_ast = Ast::new(block, None);
         self.set_body_ast(body_ast);
         Ok(())
@@ -603,10 +601,6 @@ mod tests {
     use crate::module;
     use crate::INDENT;
 
-    use wasm_bindgen_test::wasm_bindgen_test;
-
-    wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
-
     fn assert_eq_strings(lhs: Vec<impl Str>, rhs: Vec<impl Str>) {
         let lhs = lhs.iter().map(|s| s.as_ref()).collect_vec();
         let rhs = rhs.iter().map(|s| s.as_ref()).collect_vec();
@@ -621,9 +615,9 @@ mod tests {
         format!("    {line}")
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn generating_definition_to_add() {
-        let parser = Parser::new_or_panic();
+        let parser = Parser::new();
         let mut to_add = ToAdd {
             name:                     DefinitionName::new_method("Main", "add"),
             explicit_parameter_names: vec!["arg1".into(), "arg2".into()],
@@ -649,9 +643,9 @@ mod tests {
         assert_eq!(ast.repr(), "Main.add arg1 arg2 =\n        arg1 + arg2\n        arg1 - arg2");
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn definition_name_tests() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let ast = parser.parse_line_ast("Foo.Bar.baz").unwrap();
         let name = DefinitionName::from_ast(&ast).unwrap();
 
@@ -664,26 +658,26 @@ mod tests {
         assert_eq!(ast.get_traversing(&name.extended_target[1].crumbs).unwrap().repr(), "Bar");
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn definition_name_rejecting_incomplete_names() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let ast = parser.parse_line_ast("Foo. .baz").unwrap();
         assert!(DefinitionName::from_ast(&ast).is_none());
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn definition_info_name() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let ast = parser.parse_line_ast("Foo.bar a b c = baz").unwrap();
         let definition = DefinitionInfo::from_root_line_ast(&ast).unwrap();
 
         assert_eq!(definition.name.to_string(), "Foo.bar");
         assert_eq!(ast.get_traversing(&definition.name.crumbs).unwrap().repr(), "Foo.bar");
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn located_definition_args() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let ast = parser.parse_line_ast("foo bar baz = a + b + c").unwrap();
         let definition = DefinitionInfo::from_root_line_ast(&ast).unwrap();
         let (arg0, arg1) = definition.args.expect_tuple();
@@ -700,7 +694,7 @@ mod tests {
         assert_eq!(ast.get_traversing(&arg1.crumbs).unwrap(), &arg1.item);
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn match_is_not_definition() {
         let cons = Ast::cons("Foo");
         let arg = Ast::number(5);
@@ -723,28 +717,24 @@ mod tests {
         assert!(def_opt.is_some());
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn list_definition_test() {
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
 
-        // TODO [mwu]
-        //  Due to a parser bug, extension methods defining operators cannot be currently
-        //  correctly recognized. When it is fixed, the following should be also supported
-        //  and covered in test: `Int.+ a = _` and `Int.+ = _`.
-        //  Issue link: https://github.com/enso-org/enso/issues/565
         let definition_lines = vec![
             "main = _",
             "Foo.Bar.foo = _",
             "Foo.Bar.baz a b = _",
-            "+ = _",
+            "+ a = _",
+            "Int.+ a = _",
             "bar = _",
             "add a b = 50",
             "* a b = _",
         ];
         let expected_def_names_in_module =
-            vec!["main", "Foo.Bar.foo", "Foo.Bar.baz", "+", "bar", "add", "*"];
+            vec!["main", "Foo.Bar.foo", "Foo.Bar.baz", "+", "Int.+", "bar", "add", "*"];
         // In definition there are no extension methods nor arg-less definitions.
-        let expected_def_names_in_def = vec!["add", "*"];
+        let expected_def_names_in_def = vec!["+", "add", "*"];
 
         // === Program with definitions in root ===
         let program = definition_lines.join("\n");
@@ -770,7 +760,7 @@ mod tests {
         assert_eq_strings(to_names(&nested_defs), expected_def_names_in_def);
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn finding_root_definition() {
         let program_to_expected_main_pos = vec![
             ("main = bar", 0),
@@ -780,7 +770,7 @@ mod tests {
             ("foo = bar\n\nmain = bar", 2),
         ];
 
-        let parser = parser_scala::Parser::new_or_panic();
+        let parser = parser::Parser::new();
         let main_id = Id::new_plain_name("main");
         for (program, expected_line_index) in program_to_expected_main_pos {
             let module = parser.parse_module(program, default()).unwrap();
@@ -793,7 +783,7 @@ mod tests {
         }
     }
 
-    #[wasm_bindgen_test]
+    #[test]
     fn getting_nested_definition() {
         let program = r"
 main =
@@ -806,7 +796,7 @@ main =
 
     add foo bar";
 
-        let module = parser_scala::Parser::new_or_panic().parse_module(program, default()).unwrap();
+        let module = parser::Parser::new().parse_module(program, default()).unwrap();
         let check_def = |id, expected_body| {
             let definition = module::get_definition(&module, &id).unwrap();
             assert_eq!(definition.body().repr(), expected_body);