delete helix-syntax crate, move into helix-core/term

Cargo.toml

@@ -4,7 +4,6 @@ members = [
   "helix-view",
   "helix-term",
   "helix-tui",
-  "helix-syntax",
   "helix-lsp",
   "xtask",
 ]

docs/architecture.md

@@ -1,11 +1,10 @@
 
-| Crate        | Description                                            |
-| -----------  | -----------                                            |
-| helix-core   | Core editing primitives, functional.                   |
-| helix-syntax | Tree-sitter grammars                                   |
-| helix-lsp    | Language server client                                 |
-| helix-view   | UI abstractions for use in backends, imperative shell. |
-| helix-term   | Terminal UI                                            |
+| Crate        | Description                                             |
+| -----------  | -----------                                             |
+| helix-core   | Core editing primitives, functional.                    |
+| helix-lsp    | Language server client                                  |
+| helix-view   | UI abstractions for use in backends, imperative shell.  |
+| helix-term   | Terminal UI                                             |
 | helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive |
 
 

helix-core/Cargo.toml

@@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
-helix-syntax = { version = "0.6", path = "../helix-syntax" }
-
 ropey = "1.3"
 smallvec = "1.8"
 smartstring = "0.2.9"
@@ -40,5 +38,8 @@ encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
+libloading = "0.7"
+anyhow = "1"
+
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }

helix-core/src/syntax.rs

@@ -6,7 +6,9 @@ use crate::{
     Rope, RopeSlice, Tendril,
 };
 
-pub use helix_syntax::get_language;
+use anyhow::{Context, Result};
+use libloading::{Library, Symbol};
+use tree_sitter::Language;
 
 use arc_swap::{ArcSwap, Guard};
 use slotmap::{DefaultKey as LayerId, HopSlotMap};
@@ -23,6 +25,34 @@ use std::{
 use once_cell::sync::{Lazy, OnceCell};
 use serde::{Deserialize, Serialize};
 
+#[cfg(unix)]
+pub const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+pub const DYLIB_EXTENSION: &str = "dll";
+
+fn replace_dashes_with_underscores(name: &str) -> String {
+    name.replace('-', "_")
+}
+
+pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = runtime_path.join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
+    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
 fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
 where
     D: serde::Deserializer<'de>,
@@ -210,7 +240,7 @@ impl LanguageConfiguration {
                 &injections_query,
                 &locals_query,
             )
-            .unwrap(); // TODO: avoid panic
+            .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error));
 
             config.configure(scopes);
             Some(Arc::new(config))

helix-term/Cargo.toml

@@ -61,5 +61,9 @@ grep-regex = "0.1.9"
 grep-searcher = "0.1.8"
 tokio-stream = "0.1.8"
 
+# compiling grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
+
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }

helix-term/build.rs

@@ -13,5 +13,10 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }

helix-syntax/build.rs → helix-term/src/grammars.rs

@@ -6,11 +6,13 @@ use std::{
     process::Command,
 };
 
-use std::sync::mpsc::channel;
+use helix_core::syntax::DYLIB_EXTENSION;
 
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+
+pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
 
     for entry in fs::read_dir(path)? {
         let entry = entry?;
@@ -32,12 +34,6 @@ fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     Ok(dirs)
 }
 
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
 fn build_library(src_path: &Path, language: &str) -> Result<()> {
     let header_path = src_path;
     // let grammar_path = src_path.join("grammar.json");
@@ -65,7 +61,12 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
         return Ok(());
     }
     let mut config = cc::Build::new();
-    config.cpp(true).opt_level(2).cargo_metadata(false);
+    config
+        .cpp(true)
+        .opt_level(2)
+        .cargo_metadata(false)
+        .host(BUILD_TARGET)
+        .target(BUILD_TARGET);
     let compiler = config.get_compiler();
     let mut command = Command::new(compiler.path());
     command.current_dir(src_path);
@@ -148,59 +149,27 @@ fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
 
-fn build_dir(dir: &str, language: &str) {
+pub fn build_dir(dir: &str, language: &str) {
     println!("Build language {}", language);
-    if PathBuf::from("languages")
+    if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../helix-syntax/languages")
         .join(dir)
         .read_dir()
         .unwrap()
         .next()
         .is_none()
     {
         eprintln!(
-            "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
+            "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
             dir
         );
         std::process::exit(1);
     }
 
     let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("languages")
+        .join("../helix-syntax/languages")
         .join(dir)
         .join("src");
 
     build_library(&path, language).unwrap();
 }
-
-fn main() {
-    let ignore = vec![
-        "tree-sitter-typescript".to_string(),
-        "tree-sitter-ocaml".to_string(),
-    ];
-    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
-    let (tx, rx) = channel();
-
-    for dir in dirs {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let language = &dir.strip_prefix("tree-sitter-").unwrap();
-            build_dir(&dir, language);
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-    // drop(tx);
-    assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
-    build_dir("tree-sitter-typescript/tsx", "tsx");
-    build_dir("tree-sitter-typescript/typescript", "typescript");
-    build_dir("tree-sitter-ocaml/ocaml", "ocaml");
-    build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
-}

helix-term/src/lib.rs

@@ -6,6 +6,7 @@ pub mod args;
 pub mod commands;
 pub mod compositor;
 pub mod config;
+pub mod grammars;
 pub mod job;
 pub mod keymap;
 pub mod ui;