diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f18da6a4391..641db8fc0a99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,8 +14,9 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: true + + - name: Fetch grammars + run: ./scripts/grammars sync - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -53,8 +54,14 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: true + + - name: Fetch grammars (Windows) + if: startsWith(matrix.os, 'windows') + run: scripts/grammars.cmd sync + + - name: Fetch grammars (Unix) + if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') + run: scripts/grammars sync - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -97,8 +104,9 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: true + + - name: Fetch grammars + run: ./scripts/grammars sync - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -144,8 +152,9 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: true + + - name: Fetch grammars + run: ./scripts/grammars sync - name: Install stable toolchain uses: actions-rs/toolchain@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b0b7ee24498..0efb1959da49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,8 +53,14 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: true + + - name: Fetch grammars (Windows) + if: startsWith(matrix.os, 'windows') + run: scripts/grammars.cmd sync + + - name: Fetch grammars (Unix) + if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') + run: scripts/grammars sync - name: Install ${{ matrix.rust }} toolchain uses: actions-rs/toolchain@v1 @@ -114,8 +120,9 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v2 - with: - submodules: false + + - name: Fetch grammars + uses: ./scripts/grammars sync - uses: actions/download-artifact@v2 # with: diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 780d1b63eaf8..000000000000 --- a/.gitmodules +++ /dev/null @@ -1,216 +0,0 @@ -[submodule "helix-syntax/languages/tree-sitter-cpp"] - path = helix-syntax/languages/tree-sitter-cpp - url = https://github.com/tree-sitter/tree-sitter-cpp - shallow = true -[submodule "helix-syntax/languages/tree-sitter-javascript"] - path = helix-syntax/languages/tree-sitter-javascript - url = https://github.com/tree-sitter/tree-sitter-javascript - shallow = true -[submodule "helix-syntax/languages/tree-sitter-julia"] - path = helix-syntax/languages/tree-sitter-julia - url = https://github.com/tree-sitter/tree-sitter-julia - shallow = true -[submodule "helix-syntax/languages/tree-sitter-python"] - path = helix-syntax/languages/tree-sitter-python - url = https://github.com/tree-sitter/tree-sitter-python - shallow = true -[submodule "helix-syntax/languages/tree-sitter-typescript"] - path = helix-syntax/languages/tree-sitter-typescript - url = https://github.com/tree-sitter/tree-sitter-typescript - shallow = true -[submodule "helix-syntax/languages/tree-sitter-agda"] - path = helix-syntax/languages/tree-sitter-agda - url = https://github.com/tree-sitter/tree-sitter-agda - shallow = true -[submodule "helix-syntax/languages/tree-sitter-go"] - path = helix-syntax/languages/tree-sitter-go - url = https://github.com/tree-sitter/tree-sitter-go - shallow = true -[submodule "helix-syntax/languages/tree-sitter-ruby"] - path = helix-syntax/languages/tree-sitter-ruby - url = https://github.com/tree-sitter/tree-sitter-ruby - shallow = true -[submodule "helix-syntax/languages/tree-sitter-java"] - path = helix-syntax/languages/tree-sitter-java - url = https://github.com/tree-sitter/tree-sitter-java - shallow = true -[submodule "helix-syntax/languages/tree-sitter-php"] - path = helix-syntax/languages/tree-sitter-php - url = https://github.com/tree-sitter/tree-sitter-php - shallow = true -[submodule "helix-syntax/languages/tree-sitter-html"] - path = helix-syntax/languages/tree-sitter-html - url = https://github.com/tree-sitter/tree-sitter-html - shallow = true -[submodule "helix-syntax/languages/tree-sitter-scala"] - path = helix-syntax/languages/tree-sitter-scala - url = https://github.com/tree-sitter/tree-sitter-scala - shallow = true -[submodule "helix-syntax/languages/tree-sitter-bash"] - path = helix-syntax/languages/tree-sitter-bash - url = https://github.com/tree-sitter/tree-sitter-bash - shallow = true -[submodule "helix-syntax/languages/tree-sitter-rust"] - path = helix-syntax/languages/tree-sitter-rust - url = https://github.com/tree-sitter/tree-sitter-rust - shallow = true -[submodule "helix-syntax/languages/tree-sitter-json"] - path = helix-syntax/languages/tree-sitter-json - url = https://github.com/tree-sitter/tree-sitter-json - shallow = true -[submodule "helix-syntax/languages/tree-sitter-css"] - path = helix-syntax/languages/tree-sitter-css - url = https://github.com/tree-sitter/tree-sitter-css - shallow = true -[submodule "helix-syntax/languages/tree-sitter-c-sharp"] - path = helix-syntax/languages/tree-sitter-c-sharp - url = https://github.com/tree-sitter/tree-sitter-c-sharp - shallow = true -[submodule "helix-syntax/languages/tree-sitter-c"] - path = helix-syntax/languages/tree-sitter-c - url = https://github.com/tree-sitter/tree-sitter-c - shallow = true -[submodule "helix-syntax/languages/tree-sitter-haskell"] - path = helix-syntax/languages/tree-sitter-haskell - url = https://github.com/tree-sitter/tree-sitter-haskell - shallow = true -[submodule "helix-syntax/languages/tree-sitter-swift"] - path = helix-syntax/languages/tree-sitter-swift - url = https://github.com/tree-sitter/tree-sitter-swift - shallow = true -[submodule "helix-syntax/languages/tree-sitter-toml"] - path = helix-syntax/languages/tree-sitter-toml - url = https://github.com/ikatyang/tree-sitter-toml - shallow = true -[submodule "helix-syntax/languages/tree-sitter-elixir"] - path = helix-syntax/languages/tree-sitter-elixir - url = https://github.com/elixir-lang/tree-sitter-elixir - shallow = true -[submodule "helix-syntax/languages/tree-sitter-nix"] - path = helix-syntax/languages/tree-sitter-nix - url = https://github.com/cstrahan/tree-sitter-nix - shallow = true -[submodule "helix-syntax/languages/tree-sitter-latex"] - path = helix-syntax/languages/tree-sitter-latex - url = https://github.com/latex-lsp/tree-sitter-latex - shallow = true -[submodule "helix-syntax/languages/tree-sitter-ledger"] - path = helix-syntax/languages/tree-sitter-ledger - url = https://github.com/cbarrete/tree-sitter-ledger - shallow = true -[submodule "helix-syntax/languages/tree-sitter-protobuf"] - path = helix-syntax/languages/tree-sitter-protobuf - url = https://github.com/yusdacra/tree-sitter-protobuf.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-ocaml"] - path = helix-syntax/languages/tree-sitter-ocaml - url = https://github.com/tree-sitter/tree-sitter-ocaml - shallow = true -[submodule "helix-syntax/languages/tree-sitter-lua"] - path = helix-syntax/languages/tree-sitter-lua - url = https://github.com/nvim-treesitter/tree-sitter-lua - shallow = true -[submodule "helix-syntax/languages/tree-sitter-yaml"] - path = helix-syntax/languages/tree-sitter-yaml - url = https://github.com/ikatyang/tree-sitter-yaml - shallow = true -[submodule "helix-syntax/languages/tree-sitter-zig"] - path = helix-syntax/languages/tree-sitter-zig - url = https://github.com/maxxnino/tree-sitter-zig - shallow = true -[submodule "helix-syntax/languages/tree-sitter-svelte"] - path = helix-syntax/languages/tree-sitter-svelte - url = https://github.com/Himujjal/tree-sitter-svelte - shallow = true -[submodule "helix-syntax/languages/tree-sitter-vue"] - path = helix-syntax/languages/tree-sitter-vue - url = https://github.com/ikatyang/tree-sitter-vue - shallow = true -[submodule "helix-syntax/languages/tree-sitter-tsq"] - path = helix-syntax/languages/tree-sitter-tsq - url = https://github.com/tree-sitter/tree-sitter-tsq - shallow = true -[submodule "helix-syntax/languages/tree-sitter-cmake"] - path = helix-syntax/languages/tree-sitter-cmake - url = https://github.com/uyha/tree-sitter-cmake - shallow = true -[submodule "helix-syntax/languages/tree-sitter-glsl"] - path = helix-syntax/languages/tree-sitter-glsl - url = https://github.com/theHamsta/tree-sitter-glsl.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-perl"] - path = helix-syntax/languages/tree-sitter-perl - url = https://github.com/ganezdragon/tree-sitter-perl - shallow = true -[submodule "helix-syntax/languages/tree-sitter-comment"] - path = helix-syntax/languages/tree-sitter-comment - url = https://github.com/stsewd/tree-sitter-comment - shallow = true -[submodule "helix-syntax/languages/tree-sitter-wgsl"] - path = helix-syntax/languages/tree-sitter-wgsl - url = https://github.com/szebniok/tree-sitter-wgsl - shallow = true -[submodule "helix-syntax/languages/tree-sitter-llvm"] - path = helix-syntax/languages/tree-sitter-llvm - url = https://github.com/benwilliamgraham/tree-sitter-llvm - shallow = true -[submodule "helix-syntax/languages/tree-sitter-markdown"] - path = helix-syntax/languages/tree-sitter-markdown - url = https://github.com/MDeiml/tree-sitter-markdown - shallow = true -[submodule "helix-syntax/languages/tree-sitter-dart"] - path = helix-syntax/languages/tree-sitter-dart - url = https://github.com/UserNobody14/tree-sitter-dart.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-dockerfile"] - path = helix-syntax/languages/tree-sitter-dockerfile - url = https://github.com/camdencheek/tree-sitter-dockerfile.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-fish"] - path = helix-syntax/languages/tree-sitter-fish - url = https://github.com/ram02z/tree-sitter-fish - shallow = true -[submodule "helix-syntax/languages/tree-sitter-git-commit"] - path = helix-syntax/languages/tree-sitter-git-commit - url = https://github.com/the-mikedavis/tree-sitter-git-commit.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-llvm-mir"] - path = helix-syntax/languages/tree-sitter-llvm-mir - url = https://github.com/Flakebi/tree-sitter-llvm-mir.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-git-diff"] - path = helix-syntax/languages/tree-sitter-git-diff - url = https://github.com/the-mikedavis/tree-sitter-git-diff.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-tablegen"] - path = helix-syntax/languages/tree-sitter-tablegen - url = https://github.com/Flakebi/tree-sitter-tablegen - shallow = true -[submodule "helix-syntax/languages/tree-sitter-git-rebase"] - path = helix-syntax/languages/tree-sitter-git-rebase - url = https://github.com/the-mikedavis/tree-sitter-git-rebase.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-lean"] - path = helix-syntax/languages/tree-sitter-lean - url = https://github.com/Julian/tree-sitter-lean - shallow = true -[submodule "helix-syntax/languages/tree-sitter-regex"] - path = helix-syntax/languages/tree-sitter-regex - url = https://github.com/tree-sitter/tree-sitter-regex.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-make"] - path = helix-syntax/languages/tree-sitter-make - url = https://github.com/alemuller/tree-sitter-make - shallow = true -[submodule "helix-syntax/languages/tree-sitter-git-config"] - path = helix-syntax/languages/tree-sitter-git-config - url = https://github.com/the-mikedavis/tree-sitter-git-config.git - shallow = true -[submodule "helix-syntax/languages/tree-sitter-graphql"] - path = helix-syntax/languages/tree-sitter-graphql - url = https://github.com/bkegley/tree-sitter-graphql - shallow = true -[submodule "helix-syntax/languages/tree-sitter-elm"] - path = helix-syntax/languages/tree-sitter-elm - url = https://github.com/elm-tooling/tree-sitter-elm - shallow = true diff --git a/Cargo.lock b/Cargo.lock index 1f979e1d5d62..f239efceed9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -368,11 +368,12 @@ dependencies = [ name = "helix-core" version = "0.6.0" dependencies = [ + "anyhow", "arc-swap", "chrono", "encoding_rs", "etcetera", - "helix-syntax", + "libloading", "log", "once_cell", "quickcheck", @@ -409,17 +410,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "helix-syntax" -version = "0.6.0" -dependencies = [ - "anyhow", - "cc", - "libloading", - "threadpool", - "tree-sitter", -] - [[package]] name = "helix-term" version = "0.6.0" @@ -1264,7 +1254,10 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" name = "xtask" version = "0.6.0" dependencies = [ + "anyhow", + "cc", "helix-core", "helix-term", + "threadpool", "toml", ] diff --git a/Cargo.toml b/Cargo.toml index 76e3ae51a12c..8c0bd3bf494b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,6 @@ members = [ "helix-view", "helix-term", "helix-tui", - "helix-syntax", "helix-lsp", "xtask", ] diff --git a/README.md b/README.md index 71010cc824b5..dda258ffd1bb 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,10 @@ We provide packaging for various distributions, but here's a quick method to build from source. ``` -git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix +git clone https://github.com/helix-editor/helix cd helix +./scripts/grammars sync +cargo xtask build-grammars cargo install --path helix-term ``` diff --git a/book/src/guides/adding_languages.md b/book/src/guides/adding_languages.md index 5844a48eeff0..761698484856 100644 --- a/book/src/guides/adding_languages.md +++ b/book/src/guides/adding_languages.md @@ -1,22 +1,15 @@ # Adding languages -## Submodules +## Grammars manifest + +To add a new language, add a new line to `scripts/revisions.txt` with the +name of the remote and the revision. Then run -To add a new language, you should first add a tree-sitter submodule. To do this, -you can run the command -```sh -git submodule add -f helix-syntax/languages/tree-sitter- -``` -For example, to add tree-sitter-ocaml you would run -```sh -git submodule add -f https://github.com/tree-sitter/tree-sitter-ocaml helix-syntax/languages/tree-sitter-ocaml -``` -Make sure the submodule is shallow by doing ```sh -git config -f .gitmodules submodule.helix-syntax/languages/tree-sitter-.shallow true +./scripts/grammars sync ``` -or you can manually add `shallow = true` to `.gitmodules`. +To download the new grammar. ## languages.toml @@ -56,14 +49,14 @@ the last matching query supercedes the ones before it. See ## Common Issues -- If you get errors when building after switching branches, you may have to remove or update tree-sitter submodules. You can update submodules by running +- If you get errors when building after switching branches, you may have to sync your grammars. You can sync grammars by running ```sh - git submodule sync; git submodule update --init + ./scripts/grammars sync + ``` +- If a parser is segfaulting or you want to remove the parser, make sure to remove the grammar repository *and* the compiled parser in `runtime/grammar/.so`. You can remove the grammar repository by running + ```sh + ./scripts/grammars clean ``` -- Make sure to not use the `--remote` flag. To remove submodules look inside the `.gitmodules` and remove directories that are not present inside of it. - -- If a parser is segfaulting or you want to remove the parser, make sure to remove the submodule *and* the compiled parser in `runtime/grammar/.so` - - The indents query is `indents.toml`, *not* `indents.scm`. See [this](https://github.com/helix-editor/helix/issues/114) issue for more information. [treesitter-language-injection]: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection diff --git a/book/src/install.md b/book/src/install.md index 1a5a9daa96f5..e815303e4ed4 100644 --- a/book/src/install.md +++ b/book/src/install.md @@ -39,7 +39,8 @@ sudo dnf install helix ## Build from source ``` -git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix +git clone https://github.com/helix-editor/helix +./scripts/grammars sync cd helix cargo install --path helix-term ``` diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml index 9056b1f622ac..8af5cf179f7a 100644 --- a/helix-core/Cargo.toml +++ b/helix-core/Cargo.toml @@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"] [features] [dependencies] -helix-syntax = { version = "0.6", path = "../helix-syntax" } - ropey = "1.3" smallvec = "1.8" tendril = "0.4.2" @@ -40,5 +38,8 @@ encoding_rs = "0.8" chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] } +libloading = "0.7" +anyhow = "1" + [dev-dependencies] quickcheck = { version = "1", default-features = false } diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs index d6ec761003c1..1d1d4646e8b6 100644 --- a/helix-core/src/syntax.rs +++ b/helix-core/src/syntax.rs @@ -6,7 +6,9 @@ use crate::{ Rope, RopeSlice, Tendril, }; -pub use helix_syntax::get_language; +use anyhow::{Context, Result}; +use libloading::{Library, Symbol}; +use tree_sitter::Language; use arc_swap::{ArcSwap, Guard}; use slotmap::{DefaultKey as LayerId, HopSlotMap}; @@ -23,23 +25,11 @@ use std::{ use once_cell::sync::{Lazy, OnceCell}; use serde::{Deserialize, Serialize}; -fn deserialize_regex<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - Option::::deserialize(deserializer)? - .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom)) - .transpose() -} +#[cfg(unix)] +const DYLIB_EXTENSION: &str = "so"; -fn deserialize_lsp_config<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - Option::::deserialize(deserializer)? - .map(|toml| toml.try_into().map_err(serde::de::Error::custom)) - .transpose() -} +#[cfg(windows)] +const DYLIB_EXTENSION: &str = "dll"; #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -143,6 +133,46 @@ impl TextObjectQuery { } } +fn replace_dashes_with_underscores(name: &str) -> String { + name.replace('-', "_") +} + +pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result { + let name = name.to_ascii_lowercase(); + let mut library_path = runtime_path.join("grammars").join(&name); + library_path.set_extension(DYLIB_EXTENSION); + + let library = unsafe { Library::new(&library_path) } + .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?; + let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name)); + let language = unsafe { + let language_fn: Symbol Language> = library + .get(language_fn_name.as_bytes()) + .with_context(|| format!("Failed to load symbol {}", language_fn_name))?; + language_fn() + }; + std::mem::forget(library); + Ok(language) +} + +fn deserialize_regex<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer)? + .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom)) + .transpose() +} + +fn deserialize_lsp_config<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer)? + .map(|toml| toml.try_into().map_err(serde::de::Error::custom)) + .transpose() +} + fn load_runtime_file(language: &str, filename: &str) -> Result { let path = crate::RUNTIME_DIR .join("queries") @@ -209,8 +239,7 @@ impl LanguageConfiguration { &highlights_query, &injections_query, &locals_query, - ) - .unwrap(); // TODO: avoid panic + ).unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. This can happen if your grammars are out of sync - try running './scripts/grammars sync'. This query could not be parsed: {:?}", self.language_id, query_error)); config.configure(scopes); Some(Arc::new(config)) diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml deleted file mode 100644 index 855839be0022..000000000000 --- a/helix-syntax/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "helix-syntax" -version = "0.6.0" -authors = ["Blaž Hrastnik "] -edition = "2021" -license = "MPL-2.0" -description = "Tree-sitter grammars support" -categories = ["editor"] -repository = "https://github.com/helix-editor/helix" -homepage = "https://helix-editor.com" -include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"] - -[dependencies] -tree-sitter = "0.20" -libloading = "0.7" -anyhow = "1" - -[build-dependencies] -cc = { version = "1" } -threadpool = { version = "1.0" } -anyhow = "1" diff --git a/helix-syntax/README.md b/helix-syntax/README.md deleted file mode 100644 index bba2197a3d2e..000000000000 --- a/helix-syntax/README.md +++ /dev/null @@ -1,13 +0,0 @@ -helix-syntax -============ - -Syntax highlighting for helix, (shallow) submodules resides here. - -Differences from nvim-treesitter --------------------------------- - -As the syntax are commonly ported from -. - -Note that we do not support the custom `#any-of` predicate which is -supported by neovim so one needs to change it to `#match` with regex. diff --git a/helix-syntax/build.rs b/helix-syntax/build.rs deleted file mode 100644 index fa8be8b380c2..000000000000 --- a/helix-syntax/build.rs +++ /dev/null @@ -1,206 +0,0 @@ -use anyhow::{anyhow, Context, Result}; -use std::fs; -use std::time::SystemTime; -use std::{ - path::{Path, PathBuf}, - process::Command, -}; - -use std::sync::mpsc::channel; - -fn collect_tree_sitter_dirs(ignore: &[String]) -> Result> { - let mut dirs = Vec::new(); - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages"); - - for entry in fs::read_dir(path)? { - let entry = entry?; - let path = entry.path(); - - if !entry.file_type()?.is_dir() { - continue; - } - - let dir = path.file_name().unwrap().to_str().unwrap().to_string(); - - // filter ignores - if ignore.contains(&dir) { - continue; - } - dirs.push(dir) - } - - Ok(dirs) -} - -#[cfg(unix)] -const DYLIB_EXTENSION: &str = "so"; - -#[cfg(windows)] -const DYLIB_EXTENSION: &str = "dll"; - -fn build_library(src_path: &Path, language: &str) -> Result<()> { - let header_path = src_path; - // let grammar_path = src_path.join("grammar.json"); - let parser_path = src_path.join("parser.c"); - let mut scanner_path = src_path.join("scanner.c"); - - let scanner_path = if scanner_path.exists() { - Some(scanner_path) - } else { - scanner_path.set_extension("cc"); - if scanner_path.exists() { - Some(scanner_path) - } else { - None - } - }; - let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars"); - let mut library_path = parser_lib_path.join(language); - library_path.set_extension(DYLIB_EXTENSION); - - let recompile = needs_recompile(&library_path, &parser_path, &scanner_path) - .with_context(|| "Failed to compare source and binary timestamps")?; - - if !recompile { - return Ok(()); - } - let mut config = cc::Build::new(); - config.cpp(true).opt_level(2).cargo_metadata(false); - let compiler = config.get_compiler(); - let mut command = Command::new(compiler.path()); - command.current_dir(src_path); - for (key, value) in compiler.env() { - command.env(key, value); - } - - if cfg!(windows) { - command - .args(&["/nologo", "/LD", "/I"]) - .arg(header_path) - .arg("/Od") - .arg("/utf-8"); - if let Some(scanner_path) = scanner_path.as_ref() { - command.arg(scanner_path); - } - - command - .arg(parser_path) - .arg("/link") - .arg(format!("/out:{}", library_path.to_str().unwrap())); - } else { - command - .arg("-shared") - .arg("-fPIC") - .arg("-fno-exceptions") - .arg("-g") - .arg("-I") - .arg(header_path) - .arg("-o") - .arg(&library_path) - .arg("-O2"); - if let Some(scanner_path) = scanner_path.as_ref() { - if scanner_path.extension() == Some("c".as_ref()) { - command.arg("-xc").arg("-std=c99").arg(scanner_path); - } else { - command.arg(scanner_path); - } - } - command.arg("-xc").arg(parser_path); - if cfg!(all(unix, not(target_os = "macos"))) { - command.arg("-Wl,-z,relro,-z,now"); - } - } - - let output = command - .output() - .with_context(|| "Failed to execute C compiler")?; - if !output.status.success() { - return Err(anyhow!( - "Parser compilation failed.\nStdout: {}\nStderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )); - } - - Ok(()) -} -fn needs_recompile( - lib_path: &Path, - parser_c_path: &Path, - scanner_path: &Option, -) -> Result { - if !lib_path.exists() { - return Ok(true); - } - let lib_mtime = mtime(lib_path)?; - if mtime(parser_c_path)? > lib_mtime { - return Ok(true); - } - if let Some(scanner_path) = scanner_path { - if mtime(scanner_path)? > lib_mtime { - return Ok(true); - } - } - Ok(false) -} - -fn mtime(path: &Path) -> Result { - Ok(fs::metadata(path)?.modified()?) -} - -fn build_dir(dir: &str, language: &str) { - println!("Build language {}", language); - if PathBuf::from("languages") - .join(dir) - .read_dir() - .unwrap() - .next() - .is_none() - { - eprintln!( - "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?", - dir - ); - std::process::exit(1); - } - - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("languages") - .join(dir) - .join("src"); - - build_library(&path, language).unwrap(); -} - -fn main() { - let ignore = vec![ - "tree-sitter-typescript".to_string(), - "tree-sitter-ocaml".to_string(), - ]; - let dirs = collect_tree_sitter_dirs(&ignore).unwrap(); - - let mut n_jobs = 0; - let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus - let (tx, rx) = channel(); - - for dir in dirs { - let tx = tx.clone(); - n_jobs += 1; - - pool.execute(move || { - let language = &dir.strip_prefix("tree-sitter-").unwrap(); - build_dir(&dir, language); - - // report progress - tx.send(1).unwrap(); - }); - } - pool.join(); - // drop(tx); - assert_eq!(rx.try_iter().sum::(), n_jobs); - - build_dir("tree-sitter-typescript/tsx", "tsx"); - build_dir("tree-sitter-typescript/typescript", "typescript"); - build_dir("tree-sitter-ocaml/ocaml", "ocaml"); - build_dir("tree-sitter-ocaml/interface", "ocaml-interface") -} diff --git a/helix-syntax/languages/tree-sitter-agda b/helix-syntax/languages/tree-sitter-agda deleted file mode 160000 index ca69cdf485e9..000000000000 --- a/helix-syntax/languages/tree-sitter-agda +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ca69cdf485e9ce2b2ef0991a720aa88d87d30231 diff --git a/helix-syntax/languages/tree-sitter-bash b/helix-syntax/languages/tree-sitter-bash deleted file mode 160000 index a8eb5cb57c66..000000000000 --- a/helix-syntax/languages/tree-sitter-bash +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a8eb5cb57c66f74c63ab950de081207cccf52017 diff --git a/helix-syntax/languages/tree-sitter-c b/helix-syntax/languages/tree-sitter-c deleted file mode 160000 index f05e279aedde..000000000000 --- a/helix-syntax/languages/tree-sitter-c +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f05e279aedde06a25801c3f2b2cc8ac17fac52ae diff --git a/helix-syntax/languages/tree-sitter-c-sharp b/helix-syntax/languages/tree-sitter-c-sharp deleted file mode 160000 index 53a65a908167..000000000000 --- a/helix-syntax/languages/tree-sitter-c-sharp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 53a65a908167d6556e1fcdb67f1ee62aac101dda diff --git a/helix-syntax/languages/tree-sitter-cmake b/helix-syntax/languages/tree-sitter-cmake deleted file mode 160000 index f6616f1e417e..000000000000 --- a/helix-syntax/languages/tree-sitter-cmake +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f6616f1e417ee8b62daf251aa1daa5d73781c596 diff --git a/helix-syntax/languages/tree-sitter-comment b/helix-syntax/languages/tree-sitter-comment deleted file mode 160000 index 5dd3c62f1bbe..000000000000 --- a/helix-syntax/languages/tree-sitter-comment +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5dd3c62f1bbe378b220fe16b317b85247898639e diff --git a/helix-syntax/languages/tree-sitter-cpp b/helix-syntax/languages/tree-sitter-cpp deleted file mode 160000 index e8dcc9d2b404..000000000000 --- a/helix-syntax/languages/tree-sitter-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8dcc9d2b404c542fd236ea5f7208f90be8a6e89 diff --git a/helix-syntax/languages/tree-sitter-css b/helix-syntax/languages/tree-sitter-css deleted file mode 160000 index 94e10230939e..000000000000 --- a/helix-syntax/languages/tree-sitter-css +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 94e10230939e702b4fa3fa2cb5c3bc7173b95d07 diff --git a/helix-syntax/languages/tree-sitter-dart b/helix-syntax/languages/tree-sitter-dart deleted file mode 160000 index 6a25376685d1..000000000000 --- a/helix-syntax/languages/tree-sitter-dart +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6a25376685d1d47968c2cef06d4db8d84a70025e diff --git a/helix-syntax/languages/tree-sitter-dockerfile b/helix-syntax/languages/tree-sitter-dockerfile deleted file mode 160000 index 7af32bc04a66..000000000000 --- a/helix-syntax/languages/tree-sitter-dockerfile +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7af32bc04a66ab196f5b9f92ac471f29372ae2ce diff --git a/helix-syntax/languages/tree-sitter-elixir b/helix-syntax/languages/tree-sitter-elixir deleted file mode 160000 index f5d7bda543da..000000000000 --- a/helix-syntax/languages/tree-sitter-elixir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f5d7bda543da788bd507b05bd722627dde66c9ec diff --git a/helix-syntax/languages/tree-sitter-elm b/helix-syntax/languages/tree-sitter-elm deleted file mode 160000 index bd50ccf66b42..000000000000 --- a/helix-syntax/languages/tree-sitter-elm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd50ccf66b42c55252ac8efc1086af4ac6bab8cd diff --git a/helix-syntax/languages/tree-sitter-fish b/helix-syntax/languages/tree-sitter-fish deleted file mode 160000 index 04e54ab6585d..000000000000 --- a/helix-syntax/languages/tree-sitter-fish +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 04e54ab6585dfd4fee6ddfe5849af56f101b6d4f diff --git a/helix-syntax/languages/tree-sitter-git-commit b/helix-syntax/languages/tree-sitter-git-commit deleted file mode 160000 index 066e395e1107..000000000000 --- a/helix-syntax/languages/tree-sitter-git-commit +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 066e395e1107df17183cf3ae4230f1a1406cc972 diff --git a/helix-syntax/languages/tree-sitter-git-config b/helix-syntax/languages/tree-sitter-git-config deleted file mode 160000 index 0e4f0baf90b5..000000000000 --- a/helix-syntax/languages/tree-sitter-git-config +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0e4f0baf90b57e5aeb62dcdbf03062c6315d43ea diff --git a/helix-syntax/languages/tree-sitter-git-diff b/helix-syntax/languages/tree-sitter-git-diff deleted file mode 160000 index c12e6ecb5448..000000000000 --- a/helix-syntax/languages/tree-sitter-git-diff +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c12e6ecb54485f764250556ffd7ccb18f8e2942b diff --git a/helix-syntax/languages/tree-sitter-git-rebase b/helix-syntax/languages/tree-sitter-git-rebase deleted file mode 160000 index 332dc528f270..000000000000 --- a/helix-syntax/languages/tree-sitter-git-rebase +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 332dc528f27044bc4427024dbb33e6941fc131f2 diff --git a/helix-syntax/languages/tree-sitter-glsl b/helix-syntax/languages/tree-sitter-glsl deleted file mode 160000 index 88408ffc5e27..000000000000 --- a/helix-syntax/languages/tree-sitter-glsl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 88408ffc5e27abcffced7010fc77396ae3636d7e diff --git a/helix-syntax/languages/tree-sitter-go b/helix-syntax/languages/tree-sitter-go deleted file mode 160000 index 0fa917a7022d..000000000000 --- a/helix-syntax/languages/tree-sitter-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0fa917a7022d1cd2e9b779a6a8fc5dc7fad69c75 diff --git a/helix-syntax/languages/tree-sitter-graphql b/helix-syntax/languages/tree-sitter-graphql deleted file mode 160000 index 5e66e961eee4..000000000000 --- a/helix-syntax/languages/tree-sitter-graphql +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5e66e961eee421786bdda8495ed1db045e06b5fe diff --git a/helix-syntax/languages/tree-sitter-haskell b/helix-syntax/languages/tree-sitter-haskell deleted file mode 160000 index b6ec26f181dd..000000000000 --- a/helix-syntax/languages/tree-sitter-haskell +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b6ec26f181dd059eedd506fa5fbeae1b8e5556c8 diff --git a/helix-syntax/languages/tree-sitter-html b/helix-syntax/languages/tree-sitter-html deleted file mode 160000 index d93af487cc75..000000000000 --- a/helix-syntax/languages/tree-sitter-html +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d93af487cc75120c89257195e6be46c999c6ba18 diff --git a/helix-syntax/languages/tree-sitter-java b/helix-syntax/languages/tree-sitter-java deleted file mode 160000 index bd6186c24d5e..000000000000 --- a/helix-syntax/languages/tree-sitter-java +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd6186c24d5eb13b4623efac9d944dcc095c0dad diff --git a/helix-syntax/languages/tree-sitter-javascript b/helix-syntax/languages/tree-sitter-javascript deleted file mode 160000 index 4a95461c4761..000000000000 --- a/helix-syntax/languages/tree-sitter-javascript +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a95461c4761c624f2263725aca79eeaefd36cad diff --git a/helix-syntax/languages/tree-sitter-json b/helix-syntax/languages/tree-sitter-json deleted file mode 160000 index 65bceef69c3b..000000000000 --- a/helix-syntax/languages/tree-sitter-json +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 65bceef69c3b0f24c0b19ce67d79f57c96e90fcb diff --git a/helix-syntax/languages/tree-sitter-julia b/helix-syntax/languages/tree-sitter-julia deleted file mode 160000 index 12ea59726212..000000000000 --- a/helix-syntax/languages/tree-sitter-julia +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 12ea597262125fc22fd2e91aa953ac69b19c26ca diff --git a/helix-syntax/languages/tree-sitter-latex b/helix-syntax/languages/tree-sitter-latex deleted file mode 160000 index 7f720661de53..000000000000 --- a/helix-syntax/languages/tree-sitter-latex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7f720661de5316c0f8fee956526d4002fa1086d8 diff --git a/helix-syntax/languages/tree-sitter-lean b/helix-syntax/languages/tree-sitter-lean deleted file mode 160000 index d98426109258..000000000000 --- a/helix-syntax/languages/tree-sitter-lean +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d98426109258b266e1e92358c5f11716d2e8f638 diff --git a/helix-syntax/languages/tree-sitter-ledger b/helix-syntax/languages/tree-sitter-ledger deleted file mode 160000 index 0cdeb0e51411..000000000000 --- a/helix-syntax/languages/tree-sitter-ledger +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0cdeb0e51411a3ba5493662952c3039de08939ca diff --git a/helix-syntax/languages/tree-sitter-llvm b/helix-syntax/languages/tree-sitter-llvm deleted file mode 160000 index 3b213925b9c4..000000000000 --- a/helix-syntax/languages/tree-sitter-llvm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3b213925b9c4f42c1acfe2e10bfbb438d9c6834d diff --git a/helix-syntax/languages/tree-sitter-llvm-mir b/helix-syntax/languages/tree-sitter-llvm-mir deleted file mode 160000 index 06fabca19454..000000000000 --- a/helix-syntax/languages/tree-sitter-llvm-mir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 06fabca19454b2dc00c1b211a7cb7ad0bc2585f1 diff --git a/helix-syntax/languages/tree-sitter-lua b/helix-syntax/languages/tree-sitter-lua deleted file mode 160000 index 6f5d40190ec8..000000000000 --- a/helix-syntax/languages/tree-sitter-lua +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f5d40190ec8a0aa8c8410699353d820f4f7d7a6 diff --git a/helix-syntax/languages/tree-sitter-make b/helix-syntax/languages/tree-sitter-make deleted file mode 160000 index a4b9187417d6..000000000000 --- a/helix-syntax/languages/tree-sitter-make +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a4b9187417d6be349ee5fd4b6e77b4172c6827dd diff --git a/helix-syntax/languages/tree-sitter-markdown b/helix-syntax/languages/tree-sitter-markdown deleted file mode 160000 index ad8c32917a16..000000000000 --- a/helix-syntax/languages/tree-sitter-markdown +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ad8c32917a16dfbb387d1da567bf0c3fb6fffde2 diff --git a/helix-syntax/languages/tree-sitter-nix b/helix-syntax/languages/tree-sitter-nix deleted file mode 160000 index 50f38ceab667..000000000000 --- a/helix-syntax/languages/tree-sitter-nix +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 50f38ceab667f9d482640edfee803d74f4edeba5 diff --git a/helix-syntax/languages/tree-sitter-ocaml b/helix-syntax/languages/tree-sitter-ocaml deleted file mode 160000 index 23d419ba4578..000000000000 --- a/helix-syntax/languages/tree-sitter-ocaml +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 23d419ba45789c5a47d31448061557716b02750a diff --git a/helix-syntax/languages/tree-sitter-perl b/helix-syntax/languages/tree-sitter-perl deleted file mode 160000 index 0ac2c6da562c..000000000000 --- a/helix-syntax/languages/tree-sitter-perl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0ac2c6da562c7a2c26ed7e8691d4a590f7e8b90a diff --git a/helix-syntax/languages/tree-sitter-php b/helix-syntax/languages/tree-sitter-php deleted file mode 160000 index 57f855461aee..000000000000 --- a/helix-syntax/languages/tree-sitter-php +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57f855461aeeca73bd4218754fb26b5ac143f98f diff --git a/helix-syntax/languages/tree-sitter-protobuf b/helix-syntax/languages/tree-sitter-protobuf deleted file mode 160000 index 19c211a01434..000000000000 --- a/helix-syntax/languages/tree-sitter-protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 19c211a01434d9f03efff99f85e19f967591b175 diff --git a/helix-syntax/languages/tree-sitter-python b/helix-syntax/languages/tree-sitter-python deleted file mode 160000 index d6210ceab11e..000000000000 --- a/helix-syntax/languages/tree-sitter-python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d6210ceab11e8d812d4ab59c07c81458ec6e5184 diff --git a/helix-syntax/languages/tree-sitter-regex b/helix-syntax/languages/tree-sitter-regex deleted file mode 160000 index e1cfca3c7989..000000000000 --- a/helix-syntax/languages/tree-sitter-regex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e1cfca3c79896ff79842f057ea13e529b66af636 diff --git a/helix-syntax/languages/tree-sitter-ruby b/helix-syntax/languages/tree-sitter-ruby deleted file mode 160000 index dfff673b41df..000000000000 --- a/helix-syntax/languages/tree-sitter-ruby +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dfff673b41df7fadcbb609c6338f38da3cdd018e diff --git a/helix-syntax/languages/tree-sitter-rust b/helix-syntax/languages/tree-sitter-rust deleted file mode 160000 index a360da0a29a1..000000000000 --- a/helix-syntax/languages/tree-sitter-rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a360da0a29a19c281d08295a35ecd0544d2da211 diff --git a/helix-syntax/languages/tree-sitter-scala b/helix-syntax/languages/tree-sitter-scala deleted file mode 160000 index 0a3dd53a7fc4..000000000000 --- a/helix-syntax/languages/tree-sitter-scala +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0a3dd53a7fc4b352a538397d054380aaa28be54c diff --git a/helix-syntax/languages/tree-sitter-svelte b/helix-syntax/languages/tree-sitter-svelte deleted file mode 160000 index 349a5984513b..000000000000 --- a/helix-syntax/languages/tree-sitter-svelte +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 349a5984513b4a4a9e143a6e746120c6ff6cf6ed diff --git a/helix-syntax/languages/tree-sitter-swift b/helix-syntax/languages/tree-sitter-swift deleted file mode 160000 index a22fa5e19bae..000000000000 --- a/helix-syntax/languages/tree-sitter-swift +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a22fa5e19bae50098e2252ea96cba3aba43f4c58 diff --git a/helix-syntax/languages/tree-sitter-tablegen b/helix-syntax/languages/tree-sitter-tablegen deleted file mode 160000 index 568dd8a93734..000000000000 --- a/helix-syntax/languages/tree-sitter-tablegen +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 568dd8a937347175fd58db83d4c4cdaeb6069bd2 diff --git a/helix-syntax/languages/tree-sitter-toml b/helix-syntax/languages/tree-sitter-toml deleted file mode 160000 index 7cff70bbcbbc..000000000000 --- a/helix-syntax/languages/tree-sitter-toml +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7cff70bbcbbc62001b465603ca1ea88edd668704 diff --git a/helix-syntax/languages/tree-sitter-tsq b/helix-syntax/languages/tree-sitter-tsq deleted file mode 160000 index b665659d3238..000000000000 --- a/helix-syntax/languages/tree-sitter-tsq +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b665659d3238e6036e22ed0e24935e60efb39415 diff --git a/helix-syntax/languages/tree-sitter-typescript b/helix-syntax/languages/tree-sitter-typescript deleted file mode 160000 index 3e897ea5925f..000000000000 --- a/helix-syntax/languages/tree-sitter-typescript +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3e897ea5925f037cfae2e551f8e6b12eec2a201a diff --git a/helix-syntax/languages/tree-sitter-vue b/helix-syntax/languages/tree-sitter-vue deleted file mode 160000 index 91fe2754796c..000000000000 --- a/helix-syntax/languages/tree-sitter-vue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 91fe2754796cd8fba5f229505a23fa08f3546c06 diff --git a/helix-syntax/languages/tree-sitter-wgsl b/helix-syntax/languages/tree-sitter-wgsl deleted file mode 160000 index f00ff52251ed..000000000000 --- a/helix-syntax/languages/tree-sitter-wgsl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f00ff52251edbd58f4d39c9c3204383253032c11 diff --git a/helix-syntax/languages/tree-sitter-yaml b/helix-syntax/languages/tree-sitter-yaml deleted file mode 160000 index 0e36bed17176..000000000000 --- a/helix-syntax/languages/tree-sitter-yaml +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0e36bed171768908f331ff7dff9d956bae016efb diff --git a/helix-syntax/languages/tree-sitter-zig b/helix-syntax/languages/tree-sitter-zig deleted file mode 160000 index 93331b8bd8b4..000000000000 --- a/helix-syntax/languages/tree-sitter-zig +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 93331b8bd8b4ebee2b575490b2758f16ad4e9f30 diff --git a/helix-syntax/src/lib.rs b/helix-syntax/src/lib.rs deleted file mode 100644 index b0ec48d82791..000000000000 --- a/helix-syntax/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -use anyhow::{Context, Result}; -use libloading::{Library, Symbol}; -use tree_sitter::Language; - -fn replace_dashes_with_underscores(name: &str) -> String { - name.replace('-', "_") -} -#[cfg(unix)] -const DYLIB_EXTENSION: &str = "so"; - -#[cfg(windows)] -const DYLIB_EXTENSION: &str = "dll"; - -pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result { - let name = name.to_ascii_lowercase(); - let mut library_path = runtime_path.join("grammars").join(&name); - // TODO: duplicated under build - library_path.set_extension(DYLIB_EXTENSION); - - let library = unsafe { Library::new(&library_path) } - .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?; - let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name)); - let language = unsafe { - let language_fn: Symbol Language> = library - .get(language_fn_name.as_bytes()) - .with_context(|| format!("Failed to load symbol {}", language_fn_name))?; - language_fn() - }; - std::mem::forget(library); - Ok(language) -} diff --git a/scripts/grammars b/scripts/grammars new file mode 100755 index 000000000000..4a1e452ae6cd --- /dev/null +++ b/scripts/grammars @@ -0,0 +1,129 @@ +#! /usr/bin/env bash + +set -e + +GRAMMARS_DIR="$(dirname "$0")/../helix-syntax/languages" +REMOTE_NAME="helix-origin" + +print_usage_and_exit() { + echo "Usage: $0 " + echo "" + echo "Commands:" + echo "" + echo " status Checks that each grammar is checked out at the revision in revisions.txt" + echo " sync Ensures all grammars are cloned at the revisions in revisions.txt" + echo " clean Removes all grammars from the grammars directory" + echo "" + exit 1 +} + +ensure_grammar_fetched() { + local remote_url + local grammar_dir + local revision + local current_remote_url + local current_revision + + remote_url="$1" + grammar_dir="${GRAMMARS_DIR}/$(basename "$1")" + # trim trailing whitespace from the revision + revision="$(echo -e "$2" | tr -d '[:space:]')" + + mkdir -p "${grammar_dir}" + + ( + set +e + cd "${grammar_dir}" + + if [[ ! -d .git ]]; then + git init + fi + + current_remote_url="$(git remote get-url "$REMOTE_NAME" 2>/dev/null || echo -n "")" + if [[ "${current_remote_url}" != "${remote_url}" ]]; then + git remote set-url "${REMOTE_NAME}" "${remote_url}" 2>/dev/null || \ + git remote add "${REMOTE_NAME}" "${remote_url}" + fi + + current_revision="$(git rev-parse HEAD 2>/dev/null)" + if [[ "${current_revision}" != "${revision}" ]]; then + # shallow clone at exact revision, + # supported in git on the server-side since 2.5.0 (July of 2015), enabled + # on GitHub by default + git fetch "${REMOTE_NAME}" "${revision}" --depth=1 + git checkout "${revision}" + fi + ) + set -e +} + +sync_grammars() { + local grammar + # revisions.txt has grammars separated by lines + while read -r line; do + # split each line on the space character + IFS=' ' read -ra grammar <<< "${line}" + # element 0 is the remote URL, element 1 is the revision + ensure_grammar_fetched "${grammar[0]}" "${grammar[1]}" + done <"$(dirname "$0")/./revisions.txt" +} + +remove_all_grammars() { + rm -rf "${GRAMMARS_DIR}"/tree-sitter-* +} + +show_grammar_status() { + local current_revision + local grammar_name + local are_any_out_of_date + + are_any_out_of_date=false + + set +e + while read -r line; do + IFS=' ' read -ra grammar <<< "${line}" + grammar_name="$(basename "${grammar[0]}")" + + current_revision="$( + cd "${GRAMMARS_DIR}/${grammar_name}" + git rev-parse HEAD 2>/dev/null || echo -n "" + )" + + if [[ "${current_revision}" != "${grammar[1]}" ]]; then + are_any_out_of_date=true + echo "${grammar_name} is out of date." + fi + done <"$(dirname "$0")/./revisions.txt" + + if [ "${are_any_out_of_date}" = "false" ]; then + echo "All grammars are up to date." + fi + + set -e +} + +if [[ $# -eq 0 ]]; then + print_usage_and_exit +fi + +while (($#)); do + __opt="$1" + shift + case "${__opt}" in + sync) + sync_grammars + ;; + + clean) + remove_all_grammars + ;; + + status) + show_grammar_status + ;; + + *) + print_usage_and_exit + ;; + esac +done diff --git a/scripts/grammars.cmd b/scripts/grammars.cmd new file mode 100755 index 000000000000..4dd578d242ce --- /dev/null +++ b/scripts/grammars.cmd @@ -0,0 +1,88 @@ +@echo off + +set GRAMMARS_DIR=%~dp0..\helix-syntax\languages +set REVISIONS_FILE=%~dp0\revisions.txt +set REMOTE_NAME=helix-origin + +call :do_%1% 2>nul +if errorlevel 1 call :print_usage_and_exit +exit /B + +:print_usage_and_exit + echo Usage: %~f0 ^ + echo. + echo Commands: + echo status Checks that each grammar is checked out at the revision in revisions.txt + echo sync Ensures all grammars are cloned at the revisions in revisions.txt + echo clean Removes all grammars from the grammars directory + echo. + exit /B 1 + +:ensure_grammar_fetched + setlocal + set "grammar_dir=%GRAMMARS_DIR%\%~nx1" + set remote_url=%~1 + set revision=%~2 + + if not exist %grammar_dir% ( + mkdir %grammar_dir% + ) + + pushd %grammar_dir% + if not exist .git ( + git init + ) + git remote set-url %REMOTE_NAME% %remote_url% 2>NUL + if errorlevel 1 git remote add %REMOTE_NAME% %remote_url% + git fetch %REMOTE_NAME% %revision% --depth=1 + git checkout %revision% + popd + exit /B 0 + +:check_grammar_status + setlocal + set "grammar=%~nx1" + set "grammar_dir=%GRAMMARS_DIR%\%grammar%" + set remote_url=%~1 + set expected_revision=%~2 + + pushd %grammar_dir% + for /F "tokens=*" %%r in ('git rev-parse HEAD') do ( + set current_revision=%%r + ) + popd + + if "%current_revision%" == "%expected_revision%" ( + exit /B 0 + ) else ( + endlocal + set are_any_out_of_date=true + echo %grammar% is out of date. + exit /B 1 + ) + +:do_clean + pushd %GRAMMARS_DIR% + for /D %%d in (tree-sitter-*) do ( + rmdir /s /q %%d + ) + popd + exit /B 0 + +:do_sync + for /F "tokens=1,2" %%i in (%REVISIONS_FILE%) do ( + call :ensure_grammar_fetched %%i %%j + ) + exit /B + + +:do_status + set are_any_out_of_date=false + for /F "tokens=1,2" %%i in (%REVISIONS_FILE%) do ( + call :check_grammar_status %%i %%j + ) + + if "%are_any_out_of_date%" == "false" ( + echo All grammars are up to date. + ) + exit /B diff --git a/scripts/revisions.txt b/scripts/revisions.txt new file mode 100644 index 000000000000..9688564d5c59 --- /dev/null +++ b/scripts/revisions.txt @@ -0,0 +1,54 @@ +https://github.com/tree-sitter/tree-sitter-agda ca69cdf485e9ce2b2ef0991a720aa88d87d30231 +https://github.com/tree-sitter/tree-sitter-bash a8eb5cb57c66f74c63ab950de081207cccf52017 +https://github.com/tree-sitter/tree-sitter-c f05e279aedde06a25801c3f2b2cc8ac17fac52ae +https://github.com/uyha/tree-sitter-cmake f6616f1e417ee8b62daf251aa1daa5d73781c596 +https://github.com/stsewd/tree-sitter-comment 5dd3c62f1bbe378b220fe16b317b85247898639e +https://github.com/tree-sitter/tree-sitter-cpp e8dcc9d2b404c542fd236ea5f7208f90be8a6e89 +https://github.com/tree-sitter/tree-sitter-c-sharp 53a65a908167d6556e1fcdb67f1ee62aac101dda +https://github.com/tree-sitter/tree-sitter-css 94e10230939e702b4fa3fa2cb5c3bc7173b95d07 +https://github.com/UserNobody14/tree-sitter-dart 6a25376685d1d47968c2cef06d4db8d84a70025e +https://github.com/camdencheek/tree-sitter-dockerfile 7af32bc04a66ab196f5b9f92ac471f29372ae2ce +https://github.com/elixir-lang/tree-sitter-elixir f5d7bda543da788bd507b05bd722627dde66c9ec +https://github.com/elm-tooling/tree-sitter-elm bd50ccf66b42c55252ac8efc1086af4ac6bab8cd +https://github.com/ram02z/tree-sitter-fish 04e54ab6585dfd4fee6ddfe5849af56f101b6d4f +https://github.com/the-mikedavis/tree-sitter-git-commit 066e395e1107df17183cf3ae4230f1a1406cc972 +https://github.com/the-mikedavis/tree-sitter-git-config 0e4f0baf90b57e5aeb62dcdbf03062c6315d43ea +https://github.com/the-mikedavis/tree-sitter-git-diff c12e6ecb54485f764250556ffd7ccb18f8e2942b +https://github.com/the-mikedavis/tree-sitter-git-rebase 332dc528f27044bc4427024dbb33e6941fc131f2 +https://github.com/theHamsta/tree-sitter-glsl 88408ffc5e27abcffced7010fc77396ae3636d7e +https://github.com/tree-sitter/tree-sitter-go 0fa917a7022d1cd2e9b779a6a8fc5dc7fad69c75 +https://github.com/bkegley/tree-sitter-graphql 5e66e961eee421786bdda8495ed1db045e06b5fe +https://github.com/tree-sitter/tree-sitter-haskell b6ec26f181dd059eedd506fa5fbeae1b8e5556c8 +https://github.com/tree-sitter/tree-sitter-html d93af487cc75120c89257195e6be46c999c6ba18 +https://github.com/tree-sitter/tree-sitter-java bd6186c24d5eb13b4623efac9d944dcc095c0dad +https://github.com/tree-sitter/tree-sitter-javascript 4a95461c4761c624f2263725aca79eeaefd36cad +https://github.com/tree-sitter/tree-sitter-json 65bceef69c3b0f24c0b19ce67d79f57c96e90fcb +https://github.com/tree-sitter/tree-sitter-julia 12ea597262125fc22fd2e91aa953ac69b19c26ca +https://github.com/latex-lsp/tree-sitter-latex 7f720661de5316c0f8fee956526d4002fa1086d8 +https://github.com/Julian/tree-sitter-lean d98426109258b266e1e92358c5f11716d2e8f638 +https://github.com/cbarrete/tree-sitter-ledger 0cdeb0e51411a3ba5493662952c3039de08939ca +https://github.com/benwilliamgraham/tree-sitter-llvm 3b213925b9c4f42c1acfe2e10bfbb438d9c6834d +https://github.com/Flakebi/tree-sitter-llvm-mir 06fabca19454b2dc00c1b211a7cb7ad0bc2585f1 +https://github.com/nvim-treesitter/tree-sitter-lua 6f5d40190ec8a0aa8c8410699353d820f4f7d7a6 +https://github.com/alemuller/tree-sitter-make a4b9187417d6be349ee5fd4b6e77b4172c6827dd +https://github.com/MDeiml/tree-sitter-markdown ad8c32917a16dfbb387d1da567bf0c3fb6fffde2 +https://github.com/cstrahan/tree-sitter-nix 50f38ceab667f9d482640edfee803d74f4edeba5 +https://github.com/tree-sitter/tree-sitter-ocaml 23d419ba45789c5a47d31448061557716b02750a +https://github.com/ganezdragon/tree-sitter-perl 0ac2c6da562c7a2c26ed7e8691d4a590f7e8b90a +https://github.com/tree-sitter/tree-sitter-php 57f855461aeeca73bd4218754fb26b5ac143f98f +https://github.com/yusdacra/tree-sitter-protobuf 19c211a01434d9f03efff99f85e19f967591b175 +https://github.com/tree-sitter/tree-sitter-python d6210ceab11e8d812d4ab59c07c81458ec6e5184 +https://github.com/tree-sitter/tree-sitter-regex e1cfca3c79896ff79842f057ea13e529b66af636 +https://github.com/tree-sitter/tree-sitter-ruby dfff673b41df7fadcbb609c6338f38da3cdd018e +https://github.com/tree-sitter/tree-sitter-rust a360da0a29a19c281d08295a35ecd0544d2da211 +https://github.com/tree-sitter/tree-sitter-scala 0a3dd53a7fc4b352a538397d054380aaa28be54c +https://github.com/Himujjal/tree-sitter-svelte 349a5984513b4a4a9e143a6e746120c6ff6cf6ed +https://github.com/tree-sitter/tree-sitter-swift a22fa5e19bae50098e2252ea96cba3aba43f4c58 +https://github.com/Flakebi/tree-sitter-tablegen 568dd8a937347175fd58db83d4c4cdaeb6069bd2 +https://github.com/ikatyang/tree-sitter-toml 7cff70bbcbbc62001b465603ca1ea88edd668704 +https://github.com/tree-sitter/tree-sitter-tsq b665659d3238e6036e22ed0e24935e60efb39415 +https://github.com/tree-sitter/tree-sitter-typescript 3e897ea5925f037cfae2e551f8e6b12eec2a201a +https://github.com/ikatyang/tree-sitter-vue 91fe2754796cd8fba5f229505a23fa08f3546c06 +https://github.com/szebniok/tree-sitter-wgsl f00ff52251edbd58f4d39c9c3204383253032c11 +https://github.com/ikatyang/tree-sitter-yaml 0e36bed171768908f331ff7dff9d956bae016efb +https://github.com/maxxnino/tree-sitter-zig 93331b8bd8b4ebee2b575490b2758f16ad4e9f30 diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 717530d0a45a..39334f165cbd 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,3 +9,6 @@ edition = "2021" helix-term = { version = "0.6", path = "../helix-term" } helix-core = { version = "0.6", path = "../helix-core" } toml = "0.5" +anyhow = "1" +cc = { version = "1" } +threadpool = { version = "1.0" } diff --git a/xtask/build.rs b/xtask/build.rs new file mode 100644 index 000000000000..d65934784b43 --- /dev/null +++ b/xtask/build.rs @@ -0,0 +1,11 @@ +// Writes the TARGET environment variable available to build scripts so that +// it is available to the xtask by reading from the environment. +// See: https://stackoverflow.com/a/51311222/7232773 +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + + println!("cargo:rustc-env=HOST={}", std::env::var("HOST").unwrap()); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7256653a38c1..2c7ae4b946ca 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,6 +2,189 @@ use std::{env, error::Error}; type DynError = Box; +pub mod tree_sitter_grammars { + use anyhow::{anyhow, Context, Result}; + use std::fs; + use std::time::SystemTime; + use std::{ + path::{Path, PathBuf}, + process::Command, + }; + + const TARGET: &str = env!("TARGET"); + const HOST: &str = env!("HOST"); + + #[cfg(unix)] + const DYLIB_EXTENSION: &str = "so"; + + #[cfg(windows)] + const DYLIB_EXTENSION: &str = "dll"; + + pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result> { + let mut dirs = Vec::new(); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages"); + + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + + if !entry.file_type()?.is_dir() { + continue; + } + + let dir = path.file_name().unwrap().to_str().unwrap().to_string(); + + // filter ignores + if ignore.contains(&dir) { + continue; + } + dirs.push(dir) + } + + Ok(dirs) + } + + fn build_library(src_path: &Path, language: &str) -> Result<()> { + let header_path = src_path; + // let grammar_path = src_path.join("grammar.json"); + let parser_path = src_path.join("parser.c"); + let mut scanner_path = src_path.join("scanner.c"); + + let scanner_path = if scanner_path.exists() { + Some(scanner_path) + } else { + scanner_path.set_extension("cc"); + if scanner_path.exists() { + Some(scanner_path) + } else { + None + } + }; + let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars"); + let mut library_path = parser_lib_path.join(language); + library_path.set_extension(DYLIB_EXTENSION); + + let recompile = needs_recompile(&library_path, &parser_path, &scanner_path) + .with_context(|| "Failed to compare source and binary timestamps")?; + + if !recompile { + return Ok(()); + } + let mut config = cc::Build::new(); + config + .cpp(true) + .opt_level(2) + .cargo_metadata(false) + .host(HOST) + .target(TARGET); + let compiler = config.get_compiler(); + let mut command = Command::new(compiler.path()); + command.current_dir(src_path); + for (key, value) in compiler.env() { + command.env(key, value); + } + + if cfg!(windows) { + command + .args(&["/nologo", "/LD", "/I"]) + .arg(header_path) + .arg("/Od") + .arg("/utf-8"); + if let Some(scanner_path) = scanner_path.as_ref() { + command.arg(scanner_path); + } + + command + .arg(parser_path) + .arg("/link") + .arg(format!("/out:{}", library_path.to_str().unwrap())); + } else { + command + .arg("-shared") + .arg("-fPIC") + .arg("-fno-exceptions") + .arg("-g") + .arg("-I") + .arg(header_path) + .arg("-o") + .arg(&library_path) + .arg("-O2"); + if let Some(scanner_path) = scanner_path.as_ref() { + if scanner_path.extension() == Some("c".as_ref()) { + command.arg("-xc").arg("-std=c99").arg(scanner_path); + } else { + command.arg(scanner_path); + } + } + command.arg("-xc").arg(parser_path); + if cfg!(all(unix, not(target_os = "macos"))) { + command.arg("-Wl,-z,relro,-z,now"); + } + } + + let output = command + .output() + .with_context(|| "Failed to execute C compiler")?; + if !output.status.success() { + return Err(anyhow!( + "Parser compilation failed.\nStdout: {}\nStderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + + Ok(()) + } + fn needs_recompile( + lib_path: &Path, + parser_c_path: &Path, + scanner_path: &Option, + ) -> Result { + if !lib_path.exists() { + return Ok(true); + } + let lib_mtime = mtime(lib_path)?; + if mtime(parser_c_path)? > lib_mtime { + return Ok(true); + } + if let Some(scanner_path) = scanner_path { + if mtime(scanner_path)? > lib_mtime { + return Ok(true); + } + } + Ok(false) + } + + fn mtime(path: &Path) -> Result { + Ok(fs::metadata(path)?.modified()?) + } + + pub fn build_dir(dir: &str, language: &str) { + println!("Build language {}", language); + if PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../helix-syntax/languages") + .join(dir) + .read_dir() + .unwrap() + .next() + .is_none() + { + eprintln!( + "The directory {} is empty, you probably need to use './scripts/grammars sync'?", + dir + ); + std::process::exit(1); + } + + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../helix-syntax/languages") + .join(dir) + .join("src"); + + build_library(&path, language).unwrap(); + } +} + pub mod helpers { use std::{ fmt::Display, @@ -243,7 +426,9 @@ pub mod path { pub mod tasks { use crate::md_gen; + use crate::tree_sitter_grammars; use crate::DynError; + use std::sync::mpsc::channel; pub fn docgen() -> Result<(), DynError> { use md_gen::*; @@ -252,6 +437,43 @@ pub mod tasks { Ok(()) } + pub fn build_grammars() -> Result<(), DynError> { + use tree_sitter_grammars::*; + + let ignore = vec![ + "tree-sitter-typescript".to_string(), + "tree-sitter-ocaml".to_string(), + ]; + let dirs = collect_tree_sitter_dirs(&ignore).unwrap(); + + let mut n_jobs = 0; + let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus + let (tx, rx) = channel(); + + for dir in dirs { + let tx = tx.clone(); + n_jobs += 1; + + pool.execute(move || { + let language = &dir.strip_prefix("tree-sitter-").unwrap(); + build_dir(&dir, language); + + // report progress + tx.send(1).unwrap(); + }); + } + pool.join(); + // drop(tx); + assert_eq!(rx.try_iter().sum::(), n_jobs); + + build_dir("tree-sitter-typescript/tsx", "tsx"); + build_dir("tree-sitter-typescript/typescript", "typescript"); + build_dir("tree-sitter-ocaml/ocaml", "ocaml"); + build_dir("tree-sitter-ocaml/interface", "ocaml-interface"); + + Ok(()) + } + pub fn print_help() { println!( " @@ -259,6 +481,7 @@ Usage: Run with `cargo xtask `, eg. `cargo xtask docgen`. Tasks: docgen: Generate files to be included in the mdbook output. + build-grammars: Build tree-sitter grammars. " ); } @@ -270,6 +493,7 @@ fn main() -> Result<(), DynError> { None => tasks::print_help(), Some(t) => match t.as_str() { "docgen" => tasks::docgen()?, + "build-grammars" => tasks::build_grammars()?, invalid => return Err(format!("Invalid task name: {}", invalid).into()), }, };