Improve parser contextualization (#10734)

CHANGELOG.md

@@ -18,11 +18,13 @@
 - [Renaming launcher executable to ensoup][10535]
 - [Space-precedence does not apply to value-level operators][10597]
 - [Must specify `--repl` to enable debug server][10709]
+- [Improved parser error reporting and performance][10734]
 
 [10468]: https://github.com/enso-org/enso/pull/10468
 [10535]: https://github.com/enso-org/enso/pull/10535
 [10597]: https://github.com/enso-org/enso/pull/10597
 [10709]: https://github.com/enso-org/enso/pull/10709
+[10734]: https://github.com/enso-org/enso/pull/10734
 
 #### Enso IDE
 

Cargo.toml

@@ -23,6 +23,7 @@ members = [
     "lib/rust/parser/generate-java",
     "lib/rust/parser/schema",
     "lib/rust/parser/debug",
+    "lib/rust/parser/debug/fuzz",
     "tools/language-server/logstat",
     "tools/language-server/wstest",
 ]
@@ -47,6 +48,11 @@ incremental = true
 debug = false
 debug-assertions = false
 
+[profile.fuzz]
+inherits = "release"
+debug-assertions = true
+overflow-checks = true
+
 [profile.bench]
 opt-level = 3
 lto = true

app/gui2/rust-ffi/src/lib.rs

@@ -40,14 +40,14 @@ pub fn is_ident_or_operator(code: &str) -> u32 {
 #[wasm_bindgen]
 pub fn is_numeric_literal(code: &str) -> bool {
     let parsed = PARSER.with(|parser| parser.run(code));
-    let enso_parser::syntax::tree::Variant::BodyBlock(body) = *parsed.variant else { return false };
+    let enso_parser::syntax::tree::Variant::BodyBlock(body) = parsed.variant else { return false };
     let [stmt] = &body.statements[..] else { return false };
-    stmt.expression.as_ref().map_or(false, |expr| match &*expr.variant {
+    stmt.expression.as_ref().map_or(false, |expr| match &expr.variant {
         enso_parser::syntax::tree::Variant::Number(_) => true,
         enso_parser::syntax::tree::Variant::UnaryOprApp(app) =>
             app.opr.code == "-"
                 && app.rhs.as_ref().map_or(false, |rhs| {
-                    matches!(*rhs.variant, enso_parser::syntax::tree::Variant::Number(_))
+                    matches!(rhs.variant, enso_parser::syntax::tree::Variant::Number(_))
                 }),
         _ => false,
     })

app/gui2/src/components/ComponentBrowser/__tests__/input.test.ts

@@ -209,11 +209,6 @@ const baseCases: ApplySuggestionCase[] = [
     suggestion: makeStaticMethod('Standard.Base.Data.Vector.new'),
     expected: 'Data.Vector.new ',
   },
-  {
-    code: 'Dat . V .',
-    suggestion: makeStaticMethod('Standard.Base.Data.Vector.new'),
-    expected: 'Data . Vector . new ',
-  },
   {
     code: '(type_method some_arg).Vector.',
     suggestion: makeStaticMethod('Standard.Base.Data.Vector.new'),
@@ -240,6 +235,15 @@ const baseCases: ApplySuggestionCase[] = [
     expected: 'a -> a.get ',
   },
 ]
+const simpleCases: ApplySuggestionCase[] = [
+  ...baseCases,
+  // This case would cause a syntax error if a spaced-operator suffix were added.
+  {
+    code: 'Dat . V .',
+    suggestion: makeStaticMethod('Standard.Base.Data.Vector.new'),
+    expected: 'Data . Vector . new ',
+  },
+]
 
 function makeComplexCase(prefix: string, suffix: string): ApplySuggestionCase[] {
   return Array.from(baseCases, (aCase) => {

app/gui2/src/util/__tests__/qualifiedName.test.ts

@@ -33,11 +33,10 @@ const validIdentifiers = [
   '+',
   '<=>',
   '*',
-  '.',
   '!=',
 ]
 const invalidIdentifiers = ['', '1', '1Abc', '1_', 'abA!', '$a', 'a$']
-// These are not valid identifiers but currently pass the qualified name regex: ['_', '.*']
+// These are not valid identifiers but currently pass the qualified name regex: ['_', '.*', '.']
 
 test.each(validIdentifiers)("'%s' is a valid identifier", (name) =>
   expect(unwrap(tryIdentifierOrOperatorIdentifier(name))).toStrictEqual(

app/gui2/src/util/ast/__tests__/__snapshots__/raw.test.ts.snap

@@ -1373,7 +1373,7 @@ exports[`Parsing 'foo bar=baz' 1`] = `
           "leftOffsetCodeStartUtf8": 7,
           "lengthInCodeBuffer": 1,
           "startInCodeBuffer": 7,
-          "type": "Operator",
+          "type": "AssignmentOperator",
           "whitespaceLengthInCodeBuffer": 0,
           "whitespaceStartInCodeBuffer": 7,
         },

app/gui2/src/util/ast/__tests__/raw.test.ts

@@ -122,14 +122,9 @@ test.each([
       { type: Tree.Type.Ident, repr: 'foo' },
     ],
   ],
-  ['(', [{ type: Tree.Type.Invalid, repr: '(' }]],
-  [
-    '(foo',
-    [
-      { type: Tree.Type.Invalid, repr: '(' },
-      { type: Tree.Type.Ident, repr: 'foo' },
-    ],
-  ],
+  // These are Invalid nodes, so the child is a subtree containing the whole expression.
+  ['(', [{ type: Tree.Type.Group, repr: '(' }]],
+  ['(foo', [{ type: Tree.Type.Group, repr: '(foo' }]],
 ])("Reading children of '%s'", (code, expected) => {
   const ast = parseEnsoLine(code)
   const children = Array.from(childrenAstNodes(ast))

app/gui2/src/util/ast/raw.ts

@@ -17,7 +17,6 @@ export type HasAstRange = SourceRange | RawAst.Tree | RawAst.Token
  */
 export function parseEnsoLine(code: string): RawAst.Tree {
   const block = parseEnso(code)
-  assert(block.type === RawAst.Tree.Type.BodyBlock)
   const soleExpression = tryGetSoleValue(block.statements)?.expression
   assertDefined(soleExpression)
   return soleExpression

build.sbt

@@ -639,10 +639,7 @@ lazy val rustParserTargetDirectory =
   SettingKey[File]("target directory for the Rust parser")
 
 (`syntax-rust-definition` / rustParserTargetDirectory) := {
-  // setting "debug" for release, because it isn't yet safely integrated into
-  // the parser definition
-  val versionName = if (BuildInfo.isReleaseMode) "debug" else "debug"
-  target.value / "rust" / versionName
+  target.value / "rust" / "parser-jni"
 }
 
 val generateRustParserLib =
@@ -668,31 +665,32 @@ val generateRustParserLib =
     target.foreach { t =>
       Cargo.rustUp(t, log)
     }
-    val baseArguments = Seq(
+    val profile = if (BuildInfo.isReleaseMode) "release" else "fuzz"
+    val arguments = Seq(
       "build",
       "-p",
       "enso-parser-jni",
+      "--profile",
+      profile,
       "-Z",
       "unstable-options"
     ) ++ target.map(t => Seq("--target", t)).getOrElse(Seq()) ++
       Seq(
         "--out-dir",
         (`syntax-rust-definition` / rustParserTargetDirectory).value.toString
       )
-    val adjustedArguments = baseArguments ++
-      (if (BuildInfo.isReleaseMode)
-         Seq("--release")
-       else Seq())
     val envVars = target
       .map(_ => Seq(("RUSTFLAGS", "-C target-feature=-crt-static")))
       .getOrElse(Seq())
-    Cargo.run(adjustedArguments, log, envVars)
+    Cargo.run(arguments, log, envVars)
   }
   FileTreeView.default.list(Seq(libGlob)).map(_._1.toFile)
 }
 
 `syntax-rust-definition` / generateRustParserLib / fileInputs +=
-  (`syntax-rust-definition` / baseDirectory).value.toGlob / "jni" / "src" / "*.rs"
+  (`syntax-rust-definition` / baseDirectory).value.toGlob / "jni" / "src" / ** / "*.rs"
+`syntax-rust-definition` / generateRustParserLib / fileInputs +=
+  (`syntax-rust-definition` / baseDirectory).value.toGlob / "src" / ** / "*.rs"
 
 val generateParserJavaSources = TaskKey[Seq[File]](
   "generateParserJavaSources",

build/build/src/engine.rs

@@ -247,7 +247,7 @@ impl Default for BuildConfigurationFlags {
             build_project_manager_package: false,
             build_launcher_bundle: false,
             build_project_manager_bundle: false,
-            generate_java_from_rust: true,
+            generate_java_from_rust: false,
             test_java_generated_from_rust: false,
             verify_packages: false,
         }

build/build/src/enso.rs

@@ -1,6 +1,6 @@
-use crate::engine::StandardLibraryTestsSelection;
 use crate::prelude::*;
 
+use crate::engine::StandardLibraryTestsSelection;
 use crate::paths::Paths;
 use crate::paths::ENSO_ENABLE_ASSERTIONS;
 use crate::paths::ENSO_META_TEST_ARGS;

build/build/src/project/backend.rs

@@ -155,11 +155,8 @@ impl IsTarget for Backend {
                 target_os == TARGET_OS,
                 "Enso Project Manager cannot be built on '{target_os}' for target '{TARGET_OS}'.",
             );
-            let config = BuildConfigurationFlags {
-                build_project_manager_bundle: true,
-                generate_java_from_rust: true,
-                ..default()
-            };
+            let config =
+                BuildConfigurationFlags { build_project_manager_bundle: true, ..default() };
             let context = inner.prepare_context(context, config)?;
             let artifacts = context.build().await?;
             let project_manager =

build/build/src/project/runtime.rs

@@ -43,11 +43,7 @@ impl IsTarget for Runtime {
         context: Context,
         job: WithDestination<Self::BuildInput>,
     ) -> BoxFuture<'static, Result<Self::Artifact>> {
-        let config = BuildConfigurationFlags {
-            build_engine_package: true,
-            generate_java_from_rust: true,
-            ..default()
-        };
+        let config = BuildConfigurationFlags { build_engine_package: true, ..default() };
         let this = *self;
         let WithDestination { inner, destination } = job;
         let triple = TargetTriple::new(inner.versions);

build/build/src/rust/enso_linter.rs

@@ -1,9 +1,11 @@
 use super::*;
 
+use crate::paths::generated::RepoRoot;
+
 use ide_ci::programs::cargo;
 use ide_ci::programs::Cargo;
 
-use crate::paths::generated::RepoRoot;
+
 
 const LINTER_CRATE_NAME: &str = "enso-parser-debug";
 const LINTER_BIN_NAME: &str = "check_syntax";

build/build/src/rust/parser.rs

@@ -12,7 +12,6 @@ use ide_ci::programs::Javac;
 
 
 const GENERATOR_CRATE_NAME: &str = "enso-parser-generate-java";
-const PARSER_JNI_CRATE_NAME: &str = "enso-parser-jni";
 const GENERATOR_BIN_NAME: &str = GENERATOR_CRATE_NAME;
 const TEST_GENERATOR_BIN_NAME: &str = "java-tests";
 const GENERATED_CODE_NAMESPACE: [&str; 3] = ["org", "enso", "syntax2"];
@@ -47,17 +46,8 @@ pub async fn generate_java(repo_root: &RepoRoot) -> Result {
     generate_java_to(repo_root, &output_path).await
 }
 
-fn cargo_build_parser_jni(repo_root: &Path) -> Result<Command> {
-    let mut ret = Cargo.cmd()?;
-    ret.current_dir(repo_root)
-        .apply(&cargo::Command::Build)
-        .apply(&cargo::Options::Package(PARSER_JNI_CRATE_NAME.into()));
-    Ok(ret)
-}
-
 #[context("Running self-tests for the generated Java sources failed.")]
 pub async fn run_self_tests(repo_root: &RepoRoot) -> Result {
-    cargo_build_parser_jni(repo_root)?.run_ok().await?;
     let base = &repo_root.target.generated_java;
     let lib = &repo_root.lib.rust.parser.generate_java.java;
     let package = repo_root.target.generated_java.join_iter(GENERATED_CODE_NAMESPACE);

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso

@@ -1119,9 +1119,9 @@ type Integer
        of bits in the operands.
 
        > Example
-         Computing the bitwise conjunction of 2_01101101 and 2_11110000.
+         Computing the bitwise conjunction of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_and 2_11110000
+             0b01101101.bit_and 0b11110000
     bit_and self that:Integer -> Integer = integer_bit_and self that
 
     ## GROUP Bitwise
@@ -1131,9 +1131,9 @@ type Integer
        The bitwise compliment negates the value of each bit in the operand.
 
        > Example
-         Bitwise negation of 2_0110.
+         Bitwise negation of 0b0110.
 
-             2_0110.bit_not
+             0b0110.bit_not
     bit_not self -> Integer = integer_bit_not self
 
     ## GROUP Bitwise
@@ -1148,9 +1148,9 @@ type Integer
        bits in the operands.
 
        > Example
-         Computing the bitwise disjunction of 2_01101101 and 2_11110000.
+         Computing the bitwise disjunction of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_or 2_11110000
+             0b01101101.bit_or 0b11110000
     bit_or self that:Integer -> Integer = integer_bit_or self that
 
     ## GROUP Bitwise
@@ -1164,9 +1164,9 @@ type Integer
        corresponding bits in the operands.
 
        > Example
-         Computing the bitwise exclusive or of 2_01101101 and 2_11110000.
+         Computing the bitwise exclusive or of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_xor 2_11110000
+             0b01101101.bit_xor 0b11110000
     bit_xor self that:Integer -> Integer = integer_bit_xor self that
 
     ## GROUP Bitwise